[ Oct 10 05:35:34 Enabled. ] [ Oct 10 05:35:34 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Oct 10 05:35:34 Method "start" exited with status 0. ] [ Oct 10 05:35:34 Rereading configuration. ] [ Oct 10 05:35:34 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.392650358Z","hostname":"sock","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.392790912Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.445659053Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.445701204Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.560117885Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.568592628Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:34.576715692Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.179433781Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.313662643Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.313685364Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.313768846Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.313828548Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.313836778Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.313847968Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.316920439Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:35.316927789Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.316938919Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.316949199Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.31697286Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.317063022Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.317101974Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.317126294Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.317137804Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.328228706Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.328262907Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"6288dae8-9661-43f6-860a-cae2ca73d224 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.427853198Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.823847838Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.823883719Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.823890269Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.823903619Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.823909599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.8239268Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.82393162Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.832442314Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.832457804Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"75bdb1b2-c020-4f15-b179-3702eca17305 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:35.920550052Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307704119Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307743Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307772411Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307778041Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307783181Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307790111Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307795351Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307800142Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307815842Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307820172Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307824362Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307839593Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307843473Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307847183Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307864973Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307868884Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307874314Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307878504Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307882314Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307885934Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307903114Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307916465Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307920385Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307923905Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307929005Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307932625Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307953116Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307959246Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307963436Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307967056Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307972196Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.307976016Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308004887Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308009977Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308014737Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308022437Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308027018Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308032238Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308063469Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308068249Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308073069Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308081319Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308086049Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308091489Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.30809637Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308100759Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308139781Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308145331Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308150711Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308155461Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308159441Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308164261Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308197152Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308205512Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308212642Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308217633Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308221603Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308225703Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308256754Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308261524Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308265434Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308269264Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308274284Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308279954Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308289884Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308294225Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308298365Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308308335Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.308315365Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.318507313Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:739"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.318560625Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999929218s","next_cleanup":"Instant { tv_sec: 1365, tv_nsec: 191422391 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.318575495Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3944"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.318590135Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.320706121Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.320740062Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.321055811Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.322732964Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.32330928Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.32331436Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:72","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323384562Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323434453Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323445873Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323455274Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323483344Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323492124Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323499915Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323507035Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323510575Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323514315Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323541066Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323571307Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.323916966Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b,\n sleds: {\n [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: bb59b8a8-537f-4e8d-8b20-1c885a53f692 (sled),\n rack_id: 6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324155882Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324161512Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324196703Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324208894Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324376898Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324542342Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.324616724Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: bb59b8a8-537f-4e8d-8b20-1c885a53f692 (sled), rack_id: 6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.325997401Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4241"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.326045422Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4623"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.329585085Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.334628208Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.334835363Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339243809Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339297051Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339408343Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:51992","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: bb59b8a8-537f-4e8d-8b20-1c885a53f692 (sled), rack_id: 6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339524477Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339540117Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339592688Z","hostname":"sock","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339596938Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339600928Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339604448Z","hostname":"sock","pid":1572,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:72","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339643029Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.33965262Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.33967469Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.33968145Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339706641Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339710441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339715542Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.339723332Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.353650388Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.371298833Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.388094324Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.426758202Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:391"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.426824703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.426838264Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:403"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.426846004Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 144.070015ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.430142951Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"43a766da-a1ed-40cb-8c69-ed0d9ab650ff\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"43a766da-a1ed-40cb-8c69-ed0d9ab650ff\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_6288dae8-9661-43f6-860a-cae2ca73d224/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.441031248Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_6288dae8-9661-43f6-860a-cae2ca73d224/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.465409999Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_6288dae8-9661-43f6-860a-cae2ca73d224/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.46546024Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.471087198Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:418"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.471095148Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_6288dae8-9661-43f6-860a-cae2ca73d224/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.471100179Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.557838101Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.57527331Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 160.011355ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.575891506Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06d72366-b2e3-4a16-a631-c6d08a76ea8c\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"06d72366-b2e3-4a16-a631-c6d08a76ea8c\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.736877782Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 92.825786ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.737417416Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a1e93e8-3175-4582-ba14-011bad5baff6\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a1e93e8-3175-4582-ba14-011bad5baff6\" }"} {"msg":"Mounting oxi_6288dae8-9661-43f6-860a-cae2ca73d224/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.778358934Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.791111569Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.791206302Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.791222472Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.791242103Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.831251365Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 127.924086ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.831736688Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77d880ed-d87c-416a-81b4-3802b477e574\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77d880ed-d87c-416a-81b4-3802b477e574\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.960876746Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 265.126325ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:36.961582825Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cde91733-213d-4aa6-a1fe-49cee593e10b\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cde91733-213d-4aa6-a1fe-49cee593e10b\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.227834101Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 209.659984ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.228545049Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4f37c585-7ee4-4630-96d6-b1dd53ecdf97\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4f37c585-7ee4-4630-96d6-b1dd53ecdf97\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.439625924Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 149.896073ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.440159358Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c390b00b-0cb7-4d3b-abf7-c9410861c456\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:35:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c390b00b-0cb7-4d3b-abf7-c9410861c456\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.52270944Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.528390759Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.52842285Z","hostname":"sock","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:72","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"internal-dns/src/resolver.rs:72","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.528495712Z","hostname":"sock","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:56188","producer_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:37.529795946Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"203.350945ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.530494695Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.530501055Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.530564386Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:923","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.530569937Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:72","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:37.590923325Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:37.735318684Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"402.10468ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:38.138730909Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"757.381793ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:38.898710626Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.144062646s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.03366194Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2699"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.033683821Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2917"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.033689031Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2569"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.033695391Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2616"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.033734792Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2998"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.033741102Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3040"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.035712554Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2842"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.037211263Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2610"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.037314956Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:40.037478981Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:40.137220885Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.685993674s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 55.648784ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:40.364460054Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 55.099322ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:40.551788094Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 190.627535ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:40.730769683Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 458.546208ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:41.091791393Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 624.356558ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:41.690520077Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 720.111327ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:42.432735937Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 977.75113ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:43.339459126Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.046938985s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:44.493504742Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:44.825930589Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"6.145570038s"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.024033255Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.024076766Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.024102477Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.024120137Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.024353083Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:46.024405945Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.024534888Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.02459282Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.02460161Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3944"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.02462006Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:51992","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.282842155Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.282918247Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.28304812Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.283128932Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.283139903Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.283177854Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.283221745Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.283232895Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.287914278Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4265","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.287934489Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4281"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.287940219Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.548479135Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:50497"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.548898956Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.54906247Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"36d14767-adf9-4630-977d-73a69438376b","remote_addr":"[fd00:1122:3344:101::1]:50497","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":303,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.579387428Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.579677406Z","hostname":"sock","pid":1572,"uri":"/sled-role","method":"GET","req_id":"68191a32-2046-4782-b3f4-7a793ac4c6bd","remote_addr":"[fd00:1122:3344:101::1]:51969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.581257917Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.581290628Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.581590616Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.581932165Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:885"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 9787c74e-6aa3-420c-816b-8e0ff3bd5985, pool_id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: ab797b6d-82f7-44d0-855a-9f6b162c795d, pool_id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 8eded13e-b276-4aef-a633-1076df21cef6, pool_id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 709edbaf-0985-4485-8160-95f1dac3765c, pool_id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 4088c9f7-6ae4-4605-9290-3fb99d2b52b1, pool_id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 9b77160c-75cc-49be-a6ab-51de00911763, pool_id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: c087ee3d-c5b0-4a98-b644-3b954ca09f51, pool_id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 16adf8a4-3cc5-47b1-8d78-3e9ecbe7ea6b, pool_id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 6990c7ec-8f34-4c27-a871-cfbd61ada8f0, pool_id: 58a836c6-d543-4800-a473-ba954788934d (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.620806878Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.620948212Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:64616"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.62128215Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 9787c74e-6aa3-420c-816b-8e0ff3bd5985, pool_id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: ab797b6d-82f7-44d0-855a-9f6b162c795d, pool_id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 8eded13e-b276-4aef-a633-1076df21cef6, pool_id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 709edbaf-0985-4485-8160-95f1dac3765c, pool_id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 4088c9f7-6ae4-4605-9290-3fb99d2b52b1, pool_id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 9b77160c-75cc-49be-a6ab-51de00911763, pool_id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: c087ee3d-c5b0-4a98-b644-3b954ca09f51, pool_id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 16adf8a4-3cc5-47b1-8d78-3e9ecbe7ea6b, pool_id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 6990c7ec-8f34-4c27-a871-cfbd61ada8f0, pool_id: 58a836c6-d543-4800-a473-ba954788934d (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621344462Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621364492Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621382353Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621391363Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621413074Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621422144Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621429874Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.621436254Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.774276386Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:46.774309627Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:47.036601689Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:47.581424975Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6787564051477987718"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:47.581541378Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5190925573526590407"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.043567435Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:48.048226037Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.056390752Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064340612Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064398363Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064440484Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064452694Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064462545Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064594288Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.064602638Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.100461962Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d7030c6d-0525-4dcd-b8be-f52db13bdece/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.109362806Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_d7030c6d-0525-4dcd-b8be-f52db13bdece","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.281999159Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.290315187Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.290362139Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.290372599Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.290384639Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.29040586Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.30027086Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.30030181Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.391266274Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.398274948Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.398312939Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.3983305Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.398377261Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.398399642Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.398534785Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.398545835Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.43330346Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_edc9d640-5eb4-45f0-b437-ff591df0b7c2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.440940071Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_edc9d640-5eb4-45f0-b437-ff591df0b7c2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.616190573Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.624152952Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.624194323Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.624207253Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.624217394Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.624232084Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.634183426Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.634210307Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.729017001Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737230257Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737267688Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737274439Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737280969Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737303539Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737412492Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.737420542Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.773397089Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b55008d6-96a4-429c-9bb8-b3b3e09ce0b0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.781390579Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_b55008d6-96a4-429c-9bb8-b3b3e09ce0b0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.958570151Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.966437949Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.96649143Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.96650277Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.9665113Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.966518831Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.976503453Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:48.976538464Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.070743223Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078650091Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078703302Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078713403Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078778744Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078790415Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078943339Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.078960889Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:49.082567934Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.113256632Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3a95930c-8e14-4d4b-9ab6-1b2526957b1c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.121048507Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_3a95930c-8e14-4d4b-9ab6-1b2526957b1c","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.289605412Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.297293354Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.297329305Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.297344905Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.297350616Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.297354656Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.306926138Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.306952568Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.400593462Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408332326Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408361287Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408367577Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408403038Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408411178Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408560912Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.408584463Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.442854494Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3b0115e8-2124-4ac7-ac05-50bad82d3f2b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.450110515Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_3b0115e8-2124-4ac7-ac05-50bad82d3f2b","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.624656828Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.632474684Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.632520215Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.632529435Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.632537465Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.632543616Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.643489754Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.643520814Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.736820589Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.74442353Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.74445491Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.744461671Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.744498872Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.744540133Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.744706147Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.744717097Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.781003872Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d3215eb3-715f-4880-b588-9ce6cf128e80/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.789511176Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_d3215eb3-715f-4880-b588-9ce6cf128e80","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.976078665Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.984061435Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.984098026Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.984105876Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.984113167Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.984135717Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.994446268Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:49.994475189Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.090854245Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.098879036Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.098917237Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.098924438Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.098931248Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.098945838Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.099102602Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.099110562Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:50.102523312Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.135300505Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ebafd717-7c12-4d33-8ff6-d733ba37ec12/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.143146171Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_ebafd717-7c12-4d33-8ff6-d733ba37ec12","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.316861602Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.325026747Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.325063778Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.325094789Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.325103729Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.325119789Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.336404026Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.336436487Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.450382706Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458625682Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458687364Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458696324Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458720985Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458728385Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458841448Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.458850348Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.497826174Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.505841405Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_dc34e056-a9a5-45e9-ac9e-76d0c390b4d1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.681558789Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.689273922Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.689319373Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.689327813Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.689335363Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.689340724Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.698538206Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.698572576Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"58a836c6-d543-4800-a473-ba954788934d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.787803994Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.795826146Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.795859906Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.795868947Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.795882147Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.795891927Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.796047461Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.796057232Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.832698866Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_58a836c6-d543-4800-a473-ba954788934d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:50.839677869Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:50.973339626Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"13.424260263s"} {"msg":"Looking for unencrypted datasets in oxp_58a836c6-d543-4800-a473-ba954788934d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.014083488Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022015637Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022053488Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.02213584Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022232492Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6288dae8-9661-43f6-860a-cae2ca73d224 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 75bdb1b2-c020-4f15-b179-3702eca17305 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022295634Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022303724Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022432898Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022676614Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:868"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022687764Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022693404Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:894","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022699835Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022705035Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022736676Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022746926Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:910"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022794437Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:798","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.022801818Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.02289163Z","hostname":"sock","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"1602d4eb-e91c-4490-99db-d2282bf5760a","remote_addr":"[fd00:1122:3344:101::1]:64616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4401815,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.053985958Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.054243915Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48853"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.05443847Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.05445052Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:51.104851777Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.105207716Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.105215126Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.157300297Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.157324157Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.208156615Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.208185825Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.208201686Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.208236547Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.252123082Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.252160643Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.252166373Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.30413526Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.308418413Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.312610833Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.324606899Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.330171925Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:51.334114749Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:52.10716898Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:52.855132412Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:52.855221214Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:52.855379898Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:53.338932992Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:53.816310383Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:53.816372865Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:53.816409566Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 51.229224ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:53.824789886Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:53.824835177Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:53.828936616Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 136.674171ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:53.887963679Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 57.050267ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:53.958556336Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 39.669674ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:53.965867299Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 61.940717ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.014689023Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 81.452155ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.026857233Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 217.746516ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.066074275Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 121.890696ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.152416897Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 167.926061ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.186092093Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 560.372899ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.292702809Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 552.820506ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.322049791Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 322.428918ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.418714424Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 848.443967ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.750636038Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 563.711215ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.861320081Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 681.381557ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:54.883690199Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 992.418504ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:55.434428951Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 1.421193213s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:55.576466418Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 1.410918547s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:55.60806891Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:56.025528234Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1599220585478262145"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:56.025637267Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"212813780872590247"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.389984454Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4289"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.390029805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4299","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.395336705Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4378"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 1.052747641s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:56.438974813Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.495969973Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4426"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.496001494Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4453"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.533903231Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4466"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.533941822Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4330"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.587663676Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4372"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.587694957Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.607732254Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4444"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.607759824Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4486"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.674883971Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4518"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.674919892Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4524"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.763165684Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4565"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:56.763203585Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Failed to get switch ID from MGS (retrying in 43.428246ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:56.795275419Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 84.863547ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:56.839737499Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 144.770403ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:56.926163203Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 1.271371679s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:57.005770077Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 861.7872ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:57.026255227Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 593.529533ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:57.071536518Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.449788831Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:48095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.449993177Z","hostname":"sock","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"2fc1198b-624c-42e7-80d9-ddb0c8785364","remote_addr":"[fd00:1122:3344:101::2]:48095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"200"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 804.293675ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:57.500269119Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:57.583463089Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6787564051477987718"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:35:57.583576781Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5190925573526590407"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.666181375Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.697476098Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.697591441Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.697878999Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.697890689Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.733615389Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786369408Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786420629Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786429749Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786440519Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.78645007Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.78645851Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.78646439Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.78647379Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.78648322Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786488521Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786496151Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786502531Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786509291Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786531832Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786537702Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786544792Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786550462Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786554513Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786558522Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786562453Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786566703Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786570683Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:57.786574833Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006\") failed: Property not found. retry in 1.488848519s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:57.896117816Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978\") failed: Property not found. retry in 1.278256473s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:58.287461853Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c\") failed: Property not found. retry in 1.339311563s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:35:58.312186923Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.669023616Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_internal_dns_48fe0604-2445-4d48-8848-43deb687031c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.669094498Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_internal_dns_aa64ac00-57f5-46d1-8717-bc594f468978"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.669104618Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_internal_dns_f69b6d73-c56f-4174-ae3f-c14afd4f4006"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.669112028Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.669447317Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.669727444Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1efdc26a-cc8b-4e37-9ae9-e4733fed4db5","remote_addr":"[fd00:1122:3344:101::1]:48853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8615435,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.672162208Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.718528449Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.771288397Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.817425531Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.817438931Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.847464221Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.848151319Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.84818512Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.876884145Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.877001318Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48717"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.877886882Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.877892742Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.895892525Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.895948507Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.91277203Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.912810951Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.928965616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.929002437Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.929159921Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.929224693Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.940278583Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.971931786Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.973189539Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:35:59.973196709Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.002319436Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 193, 129]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.002356547Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.002690675Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.002696515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.002979333Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.003370963Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.027151209Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.046260682Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.67438065Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:00.674491543Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 40.694327ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:00.805746257Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 108.604416ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:00.85453204Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 274.19259ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:00.993810195Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 335.436822ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:01.28017392Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 1.176302829s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:01.624710896Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 1.121223981s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:02.810169319Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 1.211363877s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:03.941308552Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15\") failed: Property not found. retry in 692.098074ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:05.16153649Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.874228993Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.874314496Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.874626974Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.874917062Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a82f1125-5f42-4207-a1bb-5f13c0e9522b","remote_addr":"[fd00:1122:3344:101::1]:48717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5997738,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.87751718Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.907054987Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.907204611Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60963"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:05.916546077Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:05.916644069Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"4b21d1c7-f740-42a0-be7f-9dec73e8c103","remote_addr":"[fd00:1122:3344:101::1]:60963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9298,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:05.919409512Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:06.000572597Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:06.000698781Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:35061"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:08.009680103Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:08.009840667Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"258fd87a-7c6d-41a0-86a4-d5cd6bfe8fa1","remote_addr":"[fd00:1122:3344:101::1]:35061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":2009041,"response_code":"500"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:08.011161012Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1599220585478262145"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:08.011312116Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"212813780872590247"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:08.01145967Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6787564051477987718"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:08.011597234Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5190925573526590407"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:08.013155495Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:08.190295466Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:08.190417969Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:55093"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:08.199276942Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:08.199405216Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"04d6544e-2e25-4194-8f86-ea8e8873cad4","remote_addr":"[fd00:1122:3344:101::1]:55093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8803,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:08.201262955Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:08.339120142Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:08.339233275Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:37171"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4f5baaf9-f5ec-464b-83d1-090f4ac17d15': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:12.34847061Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.348589923Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"3bed74d6-5583-4d1e-b676-0eb8d499c0db","remote_addr":"[fd00:1122:3344:101::1]:37171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009238,"response_code":"500"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:12.350181235Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14957622891715942509"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:12.350324029Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17380129755918157193"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:36:12.350459342Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3543620335361067425"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:12.352382123Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.597340849Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.597483972Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.606331945Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"35bd951d-29ee-49b2-8a79-bb3a1daf3c5d","remote_addr":"[fd00:1122:3344:101::1]:62286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8683,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.608323627Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:12.608331988Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.945550481Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.945647003Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:57348"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.954181468Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"16b2999a-300a-4b27-a7a6-b36571c1347e","remote_addr":"[fd00:1122:3344:101::1]:57348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8403,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:12.957835424Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:12.957843344Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.166200593Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.166318606Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.173907316Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"8261e63a-e025-4a5e-bb7d-9f9116ebc1b5","remote_addr":"[fd00:1122:3344:101::1]:34415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7430,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.175552809Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:13.175560489Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.567739109Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.567829111Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:52718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.57499981Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"64f8457e-1889-4ea5-b509-f33981ce8b94","remote_addr":"[fd00:1122:3344:101::1]:52718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7003,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:13.57654871Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:13.576554861Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.328989946Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.329083349Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:41823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.337767108Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"02f45399-2965-41e6-857f-bfb047943476","remote_addr":"[fd00:1122:3344:101::1]:41823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8554,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.340668414Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.340677094Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.428146246Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.428338161Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:47885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.448351208Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"74b4e1e2-9019-4c7f-9664-3e06cb7ac3b9","remote_addr":"[fd00:1122:3344:101::1]:47885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":19865,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1168430472, ip_addr: 69.164.213.136, stratum: 3, ref_time: 1728538574.2637246, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.451576292Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.451583362Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1299"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.480341169Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.480465653Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:47748"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.480780301Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.480790701Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.50125754Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.501312351Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.519461729Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.51951586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.537815642Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.537866023Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.591256378Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.591299969Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.650703922Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.650749353Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.700814441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.700857592Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.75208944Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.752144371Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.802409034Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.802458855Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.80263644Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.802742992Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.835155225Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.835200576Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.835209277Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.835216437Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.835223157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.83534477Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"02ec6d24-3923-4102-b904-72813fb953dc","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized)]","latency_us":354731,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:14.835530845Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"144.610603ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"02ec6d24-3923-4102-b904-72813fb953dc\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"02ec6d24-3923-4102-b904-72813fb953dc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.981912997Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.982317608Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:14.982341698Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.000614169Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.00066246Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.018501815Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.018548551Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.036016798Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.03613507Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.053575602Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.053620434Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.071410486Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.071451831Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.08792248Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.087960084Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.106632914Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.106673846Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.124959873Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.124997627Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.125196805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.125394682Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.157739709Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.15778847Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.157799935Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.157807829Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.157817101Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.157926801Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4ab0d663-200f-438a-bff7-0bf723ac8e33","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized)]","latency_us":175464,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.158109684Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"62.656836ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4ab0d663-200f-438a-bff7-0bf723ac8e33\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4ab0d663-200f-438a-bff7-0bf723ac8e33\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.222714305Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.223882657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.223898257Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.242769066Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.242815961Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.260825638Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.26086283Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.278142287Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.278189462Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.297300436Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.297347255Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.314860356Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.314903164Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.331902823Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.331946765Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.349812616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.349853492Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.367081458Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.367136556Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.367321989Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.36742147Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.398511224Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.398560153Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.398569313Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.398578116Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.398585214Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.398698626Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2fe069af-0cab-46d8-b241-7c7053329e56","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":175384,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.398878049Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"168.208737ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2fe069af-0cab-46d8-b241-7c7053329e56\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2fe069af-0cab-46d8-b241-7c7053329e56\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.567749481Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.567968272Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.567989459Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.586474238Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.58650901Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.605257671Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.605293654Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.622482604Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.622520564Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.638607654Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.638651005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.657039185Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.657081796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.674974479Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.675010583Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.692707648Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.692755442Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.711605407Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.711644164Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.711923191Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.712038666Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.743364436Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.743425526Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.743433748Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.743443105Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.743452555Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.743586227Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1696b31f-51c8-4548-8253-69fcff31df16","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":175272,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:15.743786932Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"143.25215ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1696b31f-51c8-4548-8253-69fcff31df16\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1696b31f-51c8-4548-8253-69fcff31df16\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.888427884Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.888795102Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.88882005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.907117026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.907169067Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.924015358Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.924056974Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.940817153Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.940863936Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.957342842Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.957383175Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.973832535Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.973874536Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.990265517Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:15.990307771Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.006913338Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.006950192Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.025080742Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.02513599Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.025311213Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.025413488Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.056748191Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.056862647Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.056872987Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.056882504Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.056909805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.056988669Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"0330872b-6693-4005-91e9-e23aa3bb58f3","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":167952,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.057137004Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"117.015592ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0330872b-6693-4005-91e9-e23aa3bb58f3\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0330872b-6693-4005-91e9-e23aa3bb58f3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.17580763Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.175998266Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.176013069Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.192577516Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.192620783Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.211172333Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.211280156Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.228582041Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.22862227Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.247080448Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.247122815Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.264726821Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.264762251Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.282156555Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.282197159Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.300630229Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.300661561Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.31927557Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.31932748Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.319619266Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.319746262Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.350336039Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.350387265Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.350396809Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.350404582Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.350413957Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.350531963Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"07641d99-515e-4b0b-9520-78dc24449582","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":174179,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.350773425Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"344.231545ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07641d99-515e-4b0b-9520-78dc24449582\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07641d99-515e-4b0b-9520-78dc24449582\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.697165729Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.697490591Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.697500125Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.714756273Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.714802203Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.732546951Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.732593725Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.7488562Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.748897498Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.767069553Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.767122626Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.785265147Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.785316543Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.802370838Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.802413327Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.820127318Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.820170688Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.837986415Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.838031914Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.838217975Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.838306046Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.86971736Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.869760945Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.869771128Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.869778768Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.869787862Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:16.869906974Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d1fa5bee-4a0c-40b3-bd86-6472cc93bd27","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized)]","latency_us":172142,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:16.87016457Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"371.630998ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1fa5bee-4a0c-40b3-bd86-6472cc93bd27\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d1fa5bee-4a0c-40b3-bd86-6472cc93bd27\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.243594231Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.243960832Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.243975747Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.264022238Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.264138287Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.282353753Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.282388938Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.299679784Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.29973957Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.318677595Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.318712696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.336793137Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.336835916Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.354684922Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.35471963Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.37262088Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.372657696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.389958657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.390003931Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.390207776Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.390325154Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.422102265Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.422133035Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.42213791Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.422145766Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.422152291Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.422293014Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b8021c09-bf14-4294-bf96-90260a7fa315","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized)]","latency_us":178052,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.422554813Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"358.530605ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8021c09-bf14-4294-bf96-90260a7fa315\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b8021c09-bf14-4294-bf96-90260a7fa315\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.782953764Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.783194108Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.783211067Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.804855516Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.804893186Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.823087045Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.823107428Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.840860374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.840901578Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.859163696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.859205547Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.876590025Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.876628314Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.895084171Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.895117969Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.911444318Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.911479916Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.929975339Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.930019535Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.930189348Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.930290798Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.962005785Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.962037558Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.962045677Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.96205193Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.962059917Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:17.962160917Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"6cba4c24-0b2a-4fee-97d3-3c8d3c5ab510","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized)]","latency_us":178601,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:17.962296352Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"388.227745ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6cba4c24-0b2a-4fee-97d3-3c8d3c5ab510\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6cba4c24-0b2a-4fee-97d3-3c8d3c5ab510\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.352767251Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.353062694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.353088083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.373765108Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.373805038Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.391204607Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.39123862Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.408621292Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.408657799Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.426449275Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.426479885Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.443503834Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.443534031Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.461348605Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.4613838Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.479180392Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.479221568Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.496725672Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.496772568Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.49694871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.497073757Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:18.528686555Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:18.52873944Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:18.528749069Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:18.528758173Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:18.528765551Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.528863203Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4bc6ec88-edb4-4deb-8e8a-c97862bfba4b","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":175488,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:18.529009148Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"443.209913ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4bc6ec88-edb4-4deb-8e8a-c97862bfba4b\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4bc6ec88-edb4-4deb-8e8a-c97862bfba4b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.974662433Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.975145749Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.975163177Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.992644483Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:18.992688453Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.012342541Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.012383071Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.030112066Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.030152605Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.047340521Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.047376832Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.065023018Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.065066097Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.081476437Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.081513507Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.100640071Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.100673277Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.118784225Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.118829525Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.119048409Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.119244812Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.149375479Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.149418277Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.149427549Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.149436821Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.149455581Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.14955358Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f0340afa-1f53-475c-a4e5-f590f45820d6","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":174200,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.149699488Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"674.059093ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0340afa-1f53-475c-a4e5-f590f45820d6\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0340afa-1f53-475c-a4e5-f590f45820d6\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:19.244530212Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"241.175911ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.826695908Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.826991144Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.827001879Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.84805141Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.848163787Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.86599316Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.866032489Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.884120616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.884152557Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.902063706Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.902101019Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.919847511Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.919883559Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.937404202Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.937440981Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.955054793Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.955091628Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.9722065Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.972246645Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.972518499Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:19.972651243Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.003278355Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.003316419Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.003325775Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.003335198Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.003351857Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.003487573Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"93374180-e004-47f9-8acc-e88f80554ec8","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized)]","latency_us":176203,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.003749749Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"530.732485ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"93374180-e004-47f9-8acc-e88f80554ec8\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"93374180-e004-47f9-8acc-e88f80554ec8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.53672386Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.537127495Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.537144849Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.557261265Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.55732Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.575225674Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.5752694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.593035663Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.593076492Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.61180015Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.611838289Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.630590445Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.630625753Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.648668196Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.648714754Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.666587707Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.666619508Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.686030877Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.686084504Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.686229764Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.686318013Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.71868847Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.718730714Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.718740389Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.718751396Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.718758953Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:20.718838436Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ae99b337-9c1e-4cf2-8c17-0b7d8fb842ad","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":181445,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:20.718974996Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"998.495025ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ae99b337-9c1e-4cf2-8c17-0b7d8fb842ad\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ae99b337-9c1e-4cf2-8c17-0b7d8fb842ad\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.721310083Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.721643354Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.721662967Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.740172803Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.74022253Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.758202843Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.75824672Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.775767972Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.775808915Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.793633424Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.7936722Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.81260237Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.812651946Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.830166867Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.830204734Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.848829922Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.848876816Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.867318296Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.867366476Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.867581403Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.867756613Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:21.899237152Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:21.899262663Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:21.899268194Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:21.899273988Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:21.899281142Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:21.899384326Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"0d3fded9-2768-4ec6-9d45-3de023012692","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":177504,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:21.899540255Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"751.883639ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0d3fded9-2768-4ec6-9d45-3de023012692\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0d3fded9-2768-4ec6-9d45-3de023012692\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.654280336Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.654555837Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.654567687Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.674290997Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.674339936Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.693456016Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.693492045Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.712304591Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.712350521Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.730496411Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.730535741Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.748767479Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.748807313Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.766888406Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.766927398Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.784721232Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.784765905Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.803077996Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.803126859Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.803366612Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.80346764Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:22.834596481Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:22.834635116Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:22.834642813Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:22.83464871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:22.834654804Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:22.83475586Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"38d44ace-55c6-404c-94bb-ee49e9bb621e","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized)]","latency_us":179869,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:22.83491404Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.547923123s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38d44ace-55c6-404c-94bb-ee49e9bb621e\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"38d44ace-55c6-404c-94bb-ee49e9bb621e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.388156513Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.388555421Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.388568302Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.41014509Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.41018081Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.428659073Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.428701195Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.44748028Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.447523556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.465730454Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.465772202Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.483364382Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.483380216Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.50230714Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.50234645Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.521072529Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.521115806Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.539083555Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.539135663Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.539316493Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.539428883Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:24.571170766Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:24.571211436Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:24.571218673Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:24.571227656Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:24.571234612Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:24.571318632Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"5d1c9aa8-2573-4b08-b627-c3384acbef18","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":182517,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:24.571457752Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.070751956s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d1c9aa8-2573-4b08-b627-c3384acbef18\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d1c9aa8-2573-4b08-b627-c3384acbef18\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.436375457Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59906"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.436512542Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"7e7c2eaf-a509-4efe-85ba-cbcec213cefd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59906","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":50,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.648838284Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.649175278Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.649184681Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.669515922Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.669556404Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.688547942Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.688575758Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.707050655Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.707085447Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.725452624Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.725479784Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.743721985Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.743748892Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.763138269Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.763181404Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.781006578Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.781035228Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.798689971Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.798733903Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.799012478Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.799138462Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:26.83059544Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:26.830641426Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:26.830650819Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:26.830657823Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:26.830665042Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:26.830743166Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"355740af-8dfe-48f0-87ae-30c1d8dcf5e7","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":181296,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:26.830921089Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.723476408s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"355740af-8dfe-48f0-87ae-30c1d8dcf5e7\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"355740af-8dfe-48f0-87ae-30c1d8dcf5e7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.562677338Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.562989215Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.56302712Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.583573604Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.583607298Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.602892524Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.602930335Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.620881368Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.620920931Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.63924236Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.639278238Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.657027804Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.657055638Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.67520599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.675238475Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.693684802Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.693795807Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.71254647Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.712588566Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.712755219Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.712923382Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:29.744709961Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:29.744774181Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:29.744781832Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:29.744790775Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:29.744797825Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:29.744928653Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b86a087f-6e89-4893-9b00-16cc73d421e8","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":181617,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:29.745106221Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.157676473s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b86a087f-6e89-4893-9b00-16cc73d421e8\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b86a087f-6e89-4893-9b00-16cc73d421e8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.906866194Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.908263959Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.908271205Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.928525707Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.92856919Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.94723825Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.947267586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.96495263Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.964988827Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.983878654Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:30.983914759Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.001420187Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.001460154Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.021162895Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.021200461Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.039414042Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.039453802Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.057512539Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.057556537Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.057743732Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.057839454Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:31.088944786Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:31.088980505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:31.088990415Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:31.088999481Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:31.089021231Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:31.089097734Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"03bc9e68-ecfc-498a-8d61-f8f3d8cc18f9","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized)]","latency_us":181595,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:31.089242582Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.283478859s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03bc9e68-ecfc-498a-8d61-f8f3d8cc18f9\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03bc9e68-ecfc-498a-8d61-f8f3d8cc18f9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.380050928Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.380358314Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.380369517Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.401350855Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.401387653Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.420088401Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.420116433Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.438489127Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.438514516Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.456498471Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.45652115Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.475370954Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.475398124Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.492733059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.492758832Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.511079807Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.511113989Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.529824611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.529857536Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.530014881Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.530168486Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:33.562613602Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:33.56265792Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:33.562667042Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:33.562674045Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:33.562679042Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:33.562743262Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1e559336-d2ba-49ae-9e56-94a2b5bec778","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":182106,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:33.562884257Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"4.596521262s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e559336-d2ba-49ae-9e56-94a2b5bec778\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e559336-d2ba-49ae-9e56-94a2b5bec778\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:34.562165795Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"319.402323ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.173032769Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.173212052Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.173222355Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.194486645Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.194516993Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.213549942Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.213582221Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.231822127Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.231851715Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.249690283Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.249711218Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.268601015Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.268633604Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.286288744Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.286322494Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.303756512Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.303793366Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.32090702Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.320944568Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.321175899Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.321273017Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:38.35322432Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:38.353264315Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:38.353274318Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:38.35328253Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:38.353290922Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:38.353370396Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8d75a311-feaf-4262-b5ee-362d9f0c1ecc","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":179778,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:38.353514624Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.756437906s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d75a311-feaf-4262-b5ee-362d9f0c1ecc\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d75a311-feaf-4262-b5ee-362d9f0c1ecc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.118801021Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.1191906Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.119201148Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.139352994Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.139396757Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.157084514Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.157111796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.175213715Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.175252997Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.193002221Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.193036056Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.210510716Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.210541758Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.228516823Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.228549242Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.246581703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.246613137Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.26401319Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.264052472Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.264206282Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.264293969Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:41.295431625Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:41.29547206Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:41.295481623Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:41.295492611Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:41.295502043Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:41.295585229Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"75afbd56-8f95-4f52-b3d5-ce33a3599ef7","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":176113,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:41.295721676Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"4.489165371s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75afbd56-8f95-4f52-b3d5-ce33a3599ef7\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75afbd56-8f95-4f52-b3d5-ce33a3599ef7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.798345282Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.798621617Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.798634518Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.81836455Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.818403785Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.836574289Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.836600005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.854978888Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.855029955Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.874201542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.874234927Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.89244416Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.89247658Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.910486396Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.91051891Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.927743251Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.927773402Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.945914441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.945963408Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.946177567Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.946296557Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:45.978994655Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:45.979033496Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:45.979044128Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:45.979052978Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:45.979058059Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:45.979130089Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"cd11cf90-f454-45f3-886a-c430fe6a21d7","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized)]","latency_us":180203,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:45.979252015Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"5.72354404s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cd11cf90-f454-45f3-886a-c430fe6a21d7\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cd11cf90-f454-45f3-886a-c430fe6a21d7\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:49.95704835Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.274887871s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.719870262Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.720207233Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.720218118Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.740992158Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.741021315Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.760609465Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.760638134Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.779242929Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.779355081Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.797223539Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.797248665Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.815504402Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.815545504Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.833725232Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.833751445Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.851822119Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.851851839Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.869796429Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.869851603Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.870020161Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.87011051Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:51.902475895Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:51.902519565Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:51.902529268Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:51.902537857Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:51.902547157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:51.902702173Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"71bc51e1-5157-421e-9faa-598c4065f688","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized)]","latency_us":182179,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:51.902869578Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"7.195839797s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71bc51e1-5157-421e-9faa-598c4065f688\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"71bc51e1-5157-421e-9faa-598c4065f688\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:56.552694011Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:56.552761082Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"220962c1-dfb1-40c8-96ce-e968694b814f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54759","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":28,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.119083226Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.119344382Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.119356879Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.141784822Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.141815919Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.162564888Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.162601048Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.182090002Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.182130006Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.200483013Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.200505992Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.218170462Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.218196084Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.235628789Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.235656539Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.253380273Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.253413395Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.271266978Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.271299435Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.271465977Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.271622574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:59.303194533Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:59.3032282Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:59.303234303Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:59.303241522Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:59.30324816Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:36:59.303344837Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a7233457-7765-44db-b72f-62ec0ad9b508","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":183650,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:36:59.303487932Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"10.334365907s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a7233457-7765-44db-b72f-62ec0ad9b508\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:36:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a7233457-7765-44db-b72f-62ec0ad9b508\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:06.310113811Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.667230254s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.667539461Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.667938273Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.667949879Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.689365476Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.689406746Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.707883696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.707914888Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.725221435Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.725266558Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.744399103Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.744429817Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.762233466Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.762263964Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.780905414Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.780938162Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.799339145Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.799369259Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.817197668Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.817248284Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.817465077Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.817567268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:09.849479592Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:09.849519343Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:09.849527884Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:09.849534494Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:09.849540822Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:09.84962008Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3c5a8470-237d-434f-9a54-4495afe81489","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized)]","latency_us":181432,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:09.849797544Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"11.973503236s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c5a8470-237d-434f-9a54-4495afe81489\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:37:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3c5a8470-237d-434f-9a54-4495afe81489\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.857306241Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.857585219Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.857596395Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.877833836Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.877879587Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.896998552Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.897031636Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.915160398Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.915193305Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.934018252Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.934061031Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.95211213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.9521447Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.970134538Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.970166864Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.988431994Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:21.988472026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:22.006470337Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:22.006510127Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:22.006720563Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:22.006826709Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:22.038555531Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:22.03859247Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:22.038603871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:22.038613414Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:22.03862233Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:22.038694444Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"be8149e0-a8a7-4abe-8218-8e822c6328cb","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized)]","latency_us":180782,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:22.038822098Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"13.517560877s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be8149e0-a8a7-4abe-8218-8e822c6328cb\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:37:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be8149e0-a8a7-4abe-8218-8e822c6328cb\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:23.056709722Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"4.94196934s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:26.666654272Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53632"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:26.666777539Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e7041bff-5bcb-404c-b125-38b072830503","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53632","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":46,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.593948477Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.59414742Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.594159092Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.615793503Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.615827694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.634982699Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.635012325Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.65369246Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.653742946Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.673424524Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.673451928Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.692204083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.692231018Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.710384758Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.710411027Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.728826562Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.728853891Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.747168123Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.747216697Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.74745736Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.747577185Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:35.779962282Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:35.780000916Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:35.780007536Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:35.780013686Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:35.780019695Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:35.780121511Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"13400b37-139c-46aa-b69e-ad5dcb7b8810","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized)]","latency_us":185641,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:35.780352207Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"19.568050819s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13400b37-139c-46aa-b69e-ad5dcb7b8810\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:37:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"13400b37-139c-46aa-b69e-ad5dcb7b8810\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:43.086829183Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"5.411287606s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.40258978Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.402852193Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.402862497Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.423886187Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.423921053Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.442711827Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.442740543Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.461853771Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.461882009Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.48037111Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.480399057Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.498487293Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.498514143Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.51735011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.51737052Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.535042083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.535063375Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.553634374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.553666982Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.553895625Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.554005466Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:55.586079137Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:55.586118072Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:55.586127372Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:55.586137208Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:55.586147267Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:55.586235817Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"713f8262-4006-4818-917e-3f13bf0a8fca","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\", TimeNotSynchronized), (\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\", TimeNotSynchronized), (\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\", TimeNotSynchronized), (\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\", TimeNotSynchronized), (\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\", TimeNotSynchronized)]","latency_us":183042,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:37:55.586381752Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"23.407309288s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"713f8262-4006-4818-917e-3f13bf0a8fca\", \"content-length\": \"124\", \"date\": \"Thu, 10 Oct 2024 05:37:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"713f8262-4006-4818-917e-3f13bf0a8fca\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:56.782777195Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61913"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:37:56.782969291Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f810abb9-9c51-4de2-b1ae-7b15e36409b5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61913","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:03.590233526Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"15.537557766s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.058904717Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.059167497Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.05917809Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.080596485Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.080631717Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.099338435Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.099369026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.117839622Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.117867719Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.136003667Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.136034239Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.153916277Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.153942125Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.172333506Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.172367725Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.191026909Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.191076457Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.210429738Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.210457817Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.2107339Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.210841865Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.295531466Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.295589883Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.295594702Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.295600392Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.295605952Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.38739071Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.391325867Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.395149321Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.399218991Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.404030454Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.408706809Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.413334082Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.417876438Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.421878095Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:19.42609995Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.364910129Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.365048021Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.390789476Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.391013839Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.468271997Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.468378771Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.473726038Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.473843435Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 50.935211ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:22.510883322Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.511790769Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:22.511933761Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 49.6669ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:22.543995625Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 97.122526ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:22.571837965Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 108.938166ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:22.692774647Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 160.061146ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:22.739872334Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 57.9679ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:22.79775783Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 58.348277ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:23.238959294Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 154.897318ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:23.536315652Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 265.119984ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:23.592668972Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 55.186883ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:23.69704492Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 65.784396ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:23.876946532Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 131.929887ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.190372391Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 597.131825ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.431567306Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 666.431521ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.478638171Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 226.209671ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.532190225Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 142.018051ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.654732428Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 258.034456ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.678457228Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 408.160553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.775984737Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 184.992402ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:24.813075587Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 428.220694ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.021785353Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 366.575433ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.034060748Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 923.077775ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.044911376Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 1.31523572s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.158774572Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 1.145396847s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.200163713Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 702.074457ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.414171061Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 535.880643ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.463891379Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 991.213374ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:25.980138616Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 1.431120901s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:26.010483647Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 1.08959698s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:26.127197979Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 1.409198212s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:26.357615645Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 660.586723ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:26.487159645Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:26.898123599Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34638"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:26.898275778Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"60b00a42-5c43-4492-9610-7fe855aa0bae","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34638","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 1.322964469s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:26.984878817Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 559.574979ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:27.162753376Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 1.188310575s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:27.230753305Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 1.291195761s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:27.454941124Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 1.340572625s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:27.732661692Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 735.732295ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:27.777908463Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 1.130690755s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:28.317943093Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 937.048875ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:28.427782833Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 1.156496964s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:28.522523706Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 552.087004ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:28.753757461Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 1.128619909s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:29.08206701Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 572.152394ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:29.315126732Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 1.287013066s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:29.375281845Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 609.735265ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:29.45908754Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 941.035893ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:29.690187301Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 560.27834ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:29.896530981Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 585.629888ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:30.078409236Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1\") failed: Property not found. retry in 1.265159891s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:30.220144096Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f\") failed: Property not found. retry in 982.297585ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:30.469743043Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5\") failed: Property not found. retry in 1.250361137s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:30.641638027Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3\") failed: Property not found. retry in 782.818326ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:30.674245905Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6\") failed: Property not found. retry in 1.110782102s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:30.683228231Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.919864691Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_e62a9e3e-9a76-4a99-83b3-3347bd06d2b1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.919938543Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_27d163b5-e31c-4e29-afde-38f83d213dd5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.919944513Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_789c828d-ce3e-429f-8f45-9a2bd013c8a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.919948293Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_05dc146c-c0c4-47b6-8f8b-c508076a636f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.919951863Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_b0ecc337-d183-485b-881b-a2d236bbdfd6"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.919955764Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.920361854Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.920751674Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f507860c-9b1f-4de9-a29b-a1f4f79312f0","remote_addr":"[fd00:1122:3344:101::1]:47748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":12840380,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.955117389Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:58544"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:31.986486824Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3740"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:43.817031394Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3758"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.683599063Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.683941502Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"d77d9020-8f6f-46bc-bc4f-273959a9b9de","remote_addr":"[fd00:1122:3344:101::1]:58544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":26728951,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.685060721Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59045"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.686086278Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"9ac39de5-0bc9-41c8-882d-9962bd123029","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59045","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":703,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:38:58.690427672Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"19.702001346s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f69b6d73-c56f-4174-ae3f-c14afd4f4006 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 48fe0604-2445-4d48-8848-43deb687031c (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: aa64ac00-57f5-46d1-8717-bc594f468978 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 4b94915e-f546-46d8-b5b2-fc13bfcdb3e5, kind: Service { id: 4f5baaf9-f5ec-464b-83d1-090f4ac17d15 }, name: Name(\"ntp-4f5baaf9-f5ec-464b-83d1-090f4ac17d15\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 129])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0ecc337-d183-485b-881b-a2d236bbdfd6 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 05dc146c-c0c4-47b6-8f8b-c508076a636f (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 27d163b5-e31c-4e29-afde-38f83d213dd5 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }, OmicronZoneConfig { id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 454da6a4-8739-4b6e-be7d-17732fe9c6e8, kind: Service { id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6 }, name: Name(\"external-dns-74dd5392-71d1-4b7d-8d5d-310d36142bb6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 140])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 69c8d488-1779-4220-a08e-6e85af04c2ff, kind: Service { id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d }, name: Name(\"external-dns-1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 182])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9064e1da-7499-479b-93ca-5ae4b7134452 (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 90034877-22f9-4939-9b59-843b55262729, kind: Service { id: 9064e1da-7499-479b-93ca-5ae4b7134452 }, name: Name(\"nexus-9064e1da-7499-479b-93ca-5ae4b7134452\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: db838d4d-3635-4797-be9c-a0ea76b02d91 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 6847cb1c-f166-44b9-b911-23c39ad27482, kind: Service { id: db838d4d-3635-4797-be9c-a0ea76b02d91 }, name: Name(\"nexus-db838d4d-3635-4797-be9c-a0ea76b02d91\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 204])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 58050463-80d5-4fd3-a093-6540c06ff09f (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 631bf7fb-60fd-452f-b2ea-575e055c74ab, kind: Service { id: 58050463-80d5-4fd3-a093-6540c06ff09f }, name: Name(\"nexus-58050463-80d5-4fd3-a093-6540c06ff09f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 207])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 43872013-330a-4bee-9be7-e8ea6563b8b1 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ce2d92ee-b859-4168-b649-4fc681310015 (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: a2710030-7d10-4dbf-a729-988513ff3cd7 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 6d6b8154-4ef5-4488-aa11-8fb901c62afb (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ce4b0fb6-8c7a-4870-b350-1e7205578247 (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b217041-9838-415c-9f03-9d8da13c81ce (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External } } } }, OmicronZoneConfig { id: 884d82cc-b09e-4ff3-b37c-240cb856f6ef (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6b72aea8-264e-4256-9a3a-91471eea3413 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 54de1f98-15b1-4f07-8270-1c50a9bf43b5 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External } } } }, OmicronZoneConfig { id: 00132286-3aa9-4d35-a0c9-94d5a50c6e1d (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5ab013-719c-4671-b551-3e59b72ed602 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 824ba783-eaa3-4445-bbdd-ace94ca3d49d (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f883c06-8992-4e1b-bdcd-081684009c83 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 512da74c-03b3-4339-ac9a-5ba13be4b38c (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.722763053Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.723167604Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:64353"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.724476208Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.724607932Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f69b6d73-c56f-4174-ae3f-c14afd4f4006, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.750874723Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.750917664Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 48fe0604-2445-4d48-8848-43deb687031c, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.769319388Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.769350539Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aa64ac00-57f5-46d1-8717-bc594f468978, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.788623976Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.788653057Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0ecc337-d183-485b-881b-a2d236bbdfd6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.806281161Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.806308702Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 789c828d-ce3e-429f-8f45-9a2bd013c8a3, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.825367283Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.825396394Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 05dc146c-c0c4-47b6-8f8b-c508076a636f, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.842974038Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.843000657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e62a9e3e-9a76-4a99-83b3-3347bd06d2b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.861051572Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.861078553Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 27d163b5-e31c-4e29-afde-38f83d213dd5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.878716447Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.878744608Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.989972205Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:58.990006626Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce2d92ee-b859-4168-b649-4fc681310015, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.102040154Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.102073054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce2d92ee-b859-4168-b649-4fc681310015, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b217041-9838-415c-9f03-9d8da13c81ce, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.201985993Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.202024904Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9b217041-9838-415c-9f03-9d8da13c81ce, dataset_name: DatasetName { pool_name: ZpoolName { id: d7030c6d-0525-4dcd-b8be-f52db13bdece (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 884d82cc-b09e-4ff3-b37c-240cb856f6ef, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.301503982Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.301540593Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 884d82cc-b09e-4ff3-b37c-240cb856f6ef, dataset_name: DatasetName { pool_name: ZpoolName { id: edc9d640-5eb4-45f0-b437-ff591df0b7c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6b72aea8-264e-4256-9a3a-91471eea3413, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.400955489Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.40099617Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6b72aea8-264e-4256-9a3a-91471eea3413, dataset_name: DatasetName { pool_name: ZpoolName { id: b55008d6-96a4-429c-9bb8-b3b3e09ce0b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54de1f98-15b1-4f07-8270-1c50a9bf43b5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.507234856Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.507271046Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 54de1f98-15b1-4f07-8270-1c50a9bf43b5, dataset_name: DatasetName { pool_name: ZpoolName { id: 3a95930c-8e14-4d4b-9ab6-1b2526957b1c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00132286-3aa9-4d35-a0c9-94d5a50c6e1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.622277353Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.622321024Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 00132286-3aa9-4d35-a0c9-94d5a50c6e1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b0115e8-2124-4ac7-ac05-50bad82d3f2b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5ab013-719c-4671-b551-3e59b72ed602, dataset_name: DatasetName { pool_name: ZpoolName { id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.712458986Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.712495587Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fb5ab013-719c-4671-b551-3e59b72ed602, dataset_name: DatasetName { pool_name: ZpoolName { id: d3215eb3-715f-4880-b588-9ce6cf128e80 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 824ba783-eaa3-4445-bbdd-ace94ca3d49d, dataset_name: DatasetName { pool_name: ZpoolName { id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.815560839Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.81560086Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 824ba783-eaa3-4445-bbdd-ace94ca3d49d, dataset_name: DatasetName { pool_name: ZpoolName { id: ebafd717-7c12-4d33-8ff6-d733ba37ec12 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f883c06-8992-4e1b-bdcd-081684009c83, dataset_name: DatasetName { pool_name: ZpoolName { id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.924456044Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:38:59.924498385Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f883c06-8992-4e1b-bdcd-081684009c83, dataset_name: DatasetName { pool_name: ZpoolName { id: dc34e056-a9a5-45e9-ac9e-76d0c390b4d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 512da74c-03b3-4339-ac9a-5ba13be4b38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.035200178Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.035238869Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 512da74c-03b3-4339-ac9a-5ba13be4b38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 58a836c6-d543-4800-a473-ba954788934d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.131785009Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.131953824Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.132317333Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.132701374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.248951622Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.249568519Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.24959515Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.251158571Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.253862542Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.253933553Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.321405569Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.32145394Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.321459721Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.32146425Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.503431449Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.503616094Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.503648935Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.503660395Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.503683305Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.543857793Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.54414664Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.544205462Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.557419879Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.557634455Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.563120709Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.568330786Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.573304157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.578775941Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.583930637Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.588711103Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.736112332Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.741548095Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.746949837Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.752078332Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.756790396Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.762017093Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.767185059Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.773235248Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.778692312Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.784204757Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.789578248Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.795904155Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.801066321Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.806314369Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.849420153Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.849805423Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.854755814Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.859886889Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.864551431Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.868635759Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.873847966Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.8800985Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.930006714Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:00.983245055Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.034301138Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.050217337Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.050244027Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.135383147Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 130, 121]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.135411318Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.197396689Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.199275719Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.199292119Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.255302003Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 244, 207]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.255327304Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.257115981Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.257129711Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.312487818Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 208, 204]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.312516908Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.314167502Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.314181672Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.370414182Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 234, 182]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.370441933Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.371002218Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.371014878Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.372371383Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.372383184Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.427388601Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 225, 140]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.427425332Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.427778381Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.429422575Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.429434005Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.429452065Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.49497747Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.495065672Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.495211826Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.498902433Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.498919844Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.498991105Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.499214931Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.62950658Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.635952079Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.63597814Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.63634692Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.63636459Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Configuring new Omicron zone: oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.636947336Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.729255794Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.843707036Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.846994793Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Installing Omicron zone: oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.847108236Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.863672202Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.883512064Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.89514863Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:01.971688154Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:02.052159081Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:02.073534034Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:02.165909454Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:03.989496909Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:03.98991013Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:03.989986782Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:03.993937996Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.233099618Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.233335515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.233414867Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.24834774Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.248407571Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.255111568Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.255272042Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.255469887Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.255592061Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.255725804Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.255756455Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.262820611Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.269724542Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.275957226Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.282885499Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:04.288234449Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 45.391986ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.536830401Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 33.480748ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.548156079Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 28.888514ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.558311736Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 43.402224ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.568158385Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 31.466508ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.578717763Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 27.928515ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.589116707Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 51.20367ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.597863547Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 98.722483ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.607445249Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 61.052577ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.617208046Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 63.929915ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.72226389Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 113.565713ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.777427262Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 33.06753ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.80168846Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 27.260788ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.839662209Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 117.020214ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.863805255Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 161.095825ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.898281002Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 97.249671ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:04.99969431Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 56.054976ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:14.340038625Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 117.02543ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.195892195Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 58.358471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.459288916Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 81.69526ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.496231298Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 140.758154ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.540086802Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 149.201234ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.590681103Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:15.59739816Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 89.541844ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.820923202Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 528.090994ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.85885691Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:15.861276163Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 463.2055ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:15.963048001Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 293.826694ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.122376803Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 114.314192ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.140256844Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 267.760521ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.163867565Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 255.906754ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.180901563Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 277.253382ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.195053646Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 235.223256ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.208611483Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 189.612425ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.223000601Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:16.223299549Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 163.669378ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.247015163Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:16.247383063Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 257.335803ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.277626359Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:16.293674091Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:16.294432301Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 1.082899424s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.440878884Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 796.349368ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.487803749Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 498.205601ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.506401429Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 500.681344ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.560906293Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 516.474542ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:16.831031651Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 255.61736ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:17.448916719Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 585.978808ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.263831382Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 262.377719ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.495418086Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 501.957965ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.532676306Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 1.115732709s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.702721831Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.009402578s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.722521862Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.163822498s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.744174592Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.089208622s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.764061985Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 630.38455ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.787100691Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 522.827957ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.851722472Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 528.18254ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.886465526Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.16550565s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.902554049Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 860.427315ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.918631562Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 71.216813ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.933622797Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 74.523388ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.950346297Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 48.12745ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:18.966394339Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 93.114225ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.023908742Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 59.037951ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.042347197Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 131.07301ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.057174728Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 413.71334ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.082227097Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 221.765417ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.184003135Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 237.809281ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.213565653Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 285.202308ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.239130875Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 797.704514ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.394408891Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 503.878661ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.43692612Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 1.365022192s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.485134588Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 859.130134ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.511711418Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 348.765052ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.582649314Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 935.431311ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.599768535Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 570.489864ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.616871585Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.26357221s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.750792279Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 1.257718884s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.79908706Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 795.016042ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.833083154Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.210806286s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.869373779Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 770.857112ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.922586099Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 678.095487ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.950555525Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 1.099943223s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:19.975972974Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.085370495s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.088565187Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.143970814s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.216425251Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 968.002965ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.248463544Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 1.48039316s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.455518093Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.449703367s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.557851955Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 585.915487ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.644414573Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 526.99952ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.660345632Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.27940155s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.707996266Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 1.1221263s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:20.865068439Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.283852174s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.027624067Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 786.314615ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.07107223Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 1.193802193s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.092846893Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.220008102s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.103631876Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.272081685s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.187491333Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 1.489791502s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.198607955Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.15438022s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.227471475Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 963.144522ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.242156271Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.248455381s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.37549153Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 838.096013ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.871700977Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:21.904166611Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:21.904580082Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 665.274734ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.949494634Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.48990721s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:21.999652573Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 1.150086622s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.010420737Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 563.668635ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.021881249Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 26.151577ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.164922882Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 51.516797ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.201445103Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 539.022972ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.216690594Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 213.043879ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.339554967Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 514.80439ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.348667527Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.348235966s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.357338145Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 812.261477ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.367174414Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.003197345s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.39249064Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.318589613s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.470655047Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 320.91931ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.563553002Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.195621246s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.596621792Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 586.289933ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.625927123Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.422844s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.635633258Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 665.573122ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.700633169Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 863.28665ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.719598008Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 591.210684ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.766347018Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 686.866359ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.876332202Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 795.138808ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:22.899666836Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 913.629611ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.17345341Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.112355587s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.193145248Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 1.109019207s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.223506737Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 1.001868296s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.378144426Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 941.848098ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.389649879Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.333331754s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.409960173Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.054596368s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.502230811Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 925.907102ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.576970778Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 649.84954ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.595275999Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.014229004s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.708962941Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 576.378976ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.722534828Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:23.725442995Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:23.725691181Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.111352839s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.802840071Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.037843162s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.813851341Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 71.138079ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:23.923980529Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 87.77741ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.034577269Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.048933549s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.225240526Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 793.122269ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.280080859Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 1.437658706s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.298129864Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 248.498393ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.313139769Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 831.439945ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.327115036Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.196333062s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.344584916Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 1.246934632s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.367420997Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 874.797889ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.391248504Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 502.423673ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.403139987Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 585.83231ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.516000607Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 577.559403ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.569237328Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 428.392944ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.580699259Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.0816553s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.733028997Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.464768201s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.753947368Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.076874922s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.861641332Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 937.864582ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.922462172Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.364716868s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:24.935729101Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 907.205086ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.024531378Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 787.865888ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.09340339Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 990.864956ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.11657704Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.371036305s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.173080787Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.131476587s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.188753279Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:25.190990848Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:25.191316316Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:25.191407399Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:25.191548802Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:25.201535815Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:25.333918189Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 1.296832858s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.390347494Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 878.445963ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.41033285Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 40.591266ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.501637952Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.408999384s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.556367172Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 78.269232ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.570174485Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 1.106385937s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.65433469Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 234.131738ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.69614368Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 26.056106ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.709994825Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 55.573367ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.727159746Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 1.125019433s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.757976447Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 138.910885ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.772764946Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 84.868231ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.795268419Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.313651098s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.829221922Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 931.118201ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.873445876Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 291.720661ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.893541204Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 1.386398301s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.951403407Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 247.164371ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.979139947Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 221.332459ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:25.99445328Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.193658634s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.008925421Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 812.086247ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.023048862Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 863.53098ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.123133975Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 296.38693ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.201098277Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 345.651609ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.231175638Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 1.12487872s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.243326648Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.106424652s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.258358633Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.432028032s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.304536289Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 701.666407ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.318052814Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 726.974215ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.332775322Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 978.688249ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.511379331Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.051873995s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.556695614Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 980.882872ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.588736557Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 1.160995392s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.708139779Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 661.725447ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.77543738Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 681.275005ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.819790817Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 620.233513ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.847076435Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 1.357780715s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.897647935Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 787.549062ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:26.984577113Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 1.076337566s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.000137762Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.148440671s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.031241831Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.35278316s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.073799251Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.094152377s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.155310885Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 845.097155ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.216050184Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:27.219986557Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:27.220695946Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 879.331349ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.352112144Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 686.816767ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.376417733Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 563.389048ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.388312297Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 40.185397ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.443615302Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 1.434523372s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.454325163Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 1.095118756s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.48155719Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 129.502513ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.494520731Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 1.064015531s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.50702611Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 692.018984ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.517544277Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 1.375680426s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.608999104Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.149745234s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.651023629Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 275.962206ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.660816467Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 886.879061ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.747181549Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 1.272087492s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.781621916Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 1.108556924s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.882347676Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 593.6689ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.948499147Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 761.79773ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:27.965415482Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.202190391s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.0728976Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.070888287s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.085348638Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 1.241224821s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.094380736Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.265745183s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.191726597Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 1.1722323s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.224207081Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 754.998691ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.245384419Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.390549044s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.260976719Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 976.091206ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.274045143Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.375615852s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.438803948Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 521.059314ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.553931578Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 648.262187ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.581966775Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 1.05251516s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.592345968Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.083929313s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.64676328Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 1.045898121s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.739687476Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 528.3577ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.812410019Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 1.377816996s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.902483489Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:28.920803811Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:28.945230264Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"99736d42-1cb6-4ace-988b-231993e53728","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51272","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":15841,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 1.231556159s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:28.997611932Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 1.071122338s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.00931384Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 1.213874957s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.019293223Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 931.183978ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.063807144Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.109198377s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.086541883Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.341325029s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.168959491Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 852.538095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.241403567Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 785.357958ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.263045067Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.058875517s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.285786635Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 1.148955257s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.34641049Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 571.727178ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.355635803Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 971.52833ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.407922909Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247\") failed: Property not found. retry in 1.18428421s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.467113886Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 879.644762ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.656184812Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 616.722454ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.666220756Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.468544004s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.744202337Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 1.154434658s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.798901317Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 557.838789ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.824903951Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 700.633575ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:29.93660173Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 777.336727ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.00614823Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 508.694871ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.060230783Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 1.04652157s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.093461598Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 728.824196ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.106104931Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.02385767s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.207120088Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 981.050537ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.238101053Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d\") failed: Property not found. retry in 1.386364743s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.246133205Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602\") failed: Property not found. retry in 882.768394ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.289258149Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.359688241s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.298455341Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d\") failed: Property not found. retry in 1.194255535s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.353864799Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb\") failed: Property not found. retry in 975.880103ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.389593879Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef\") failed: Property not found. retry in 1.02873778s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.401728119Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 1.110230893s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.506044224Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce\") failed: Property not found. retry in 1.410219633s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.520503634Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 712.269921ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.547243468Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83\") failed: Property not found. retry in 505.297447ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.579511237Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413\") failed: Property not found. retry in 1.456541797s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.64764941Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5\") failed: Property not found. retry in 815.469574ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.796872967Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 507.499499ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.8475444Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 1.151384602s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:30.96535272Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c\") failed: Property not found. retry in 930.889528ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.150652086Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 582.566902ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.225356871Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 1.194198238s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.23519318Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.093855566s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.244445764Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 666.177649ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.269726479Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 1.378742195s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.368242791Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7\") failed: Property not found. retry in 625.905145ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.645633921Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\") failed: Property not found. retry in 1.187093116s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.670770952Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1\") failed: Property not found. retry in 1.34641718s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.826202252Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 991.540223ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:31.965393264Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 868.336222ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:32.138050847Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.080752935s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:32.351187446Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 1.069560695s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:32.441877672Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 871.159938ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:32.76063939Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6\") failed: Property not found. retry in 617.945777ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:32.973808369Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 1.084346285s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:33.026685071Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.248742459s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:33.445884831Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 653.380952ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:33.527441217Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 1.227782899s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:33.644757234Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:33.986481926Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"79.021207213s"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 555.072124ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:34.126412428Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 1.259810139s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:34.19757766Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 752.12055ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:34.694115896Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.385489521s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:34.703584895Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 1.139669699s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:34.882419761Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452\") failed: Property not found. retry in 1.253777405s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:35.456584119Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91\") failed: Property not found. retry in 965.017555ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:35.467082596Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f\") failed: Property not found. retry in 1.38518016s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:36.032602446Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015\") failed: Property not found. retry in 1.294941714s","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:36.100520264Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445092554Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_pantry_6d6b8154-4ef5-4488-aa11-8fb901c62afb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445292999Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_oximeter_43872013-330a-4bee-9be7-e8ea6563b8b1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445299119Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_nexus_9064e1da-7499-479b-93ca-5ae4b7134452"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44530337Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_nexus_db838d4d-3635-4797-be9c-a0ea76b02d91"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445307339Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_pantry_a2710030-7d10-4dbf-a729-988513ff3cd7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44531107Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_nexus_58050463-80d5-4fd3-a093-6540c06ff09f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.4453155Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_884d82cc-b09e-4ff3-b37c-240cb856f6ef"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.4453191Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_6b72aea8-264e-4256-9a3a-91471eea3413"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44532235Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_54de1f98-15b1-4f07-8270-1c50a9bf43b5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44532603Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_00132286-3aa9-4d35-a0c9-94d5a50c6e1d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44532983Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_9b217041-9838-415c-9f03-9d8da13c81ce"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44533588Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_fb5ab013-719c-4671-b551-3e59b72ed602"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44533967Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_pantry_ce4b0fb6-8c7a-4870-b350-1e7205578247"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.44534304Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_6f883c06-8992-4e1b-bdcd-081684009c83"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445346431Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_external_dns_1dde945c-0d24-4fa1-bc15-9f5ffae3c10d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445349851Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_512da74c-03b3-4339-ac9a-5ba13be4b38c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.445353311Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_824ba783-eaa3-4445-bbdd-ace94ca3d49d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.452876649Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_external_dns_74dd5392-71d1-4b7d-8d5d-310d36142bb6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.452885529Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_clickhouse_ce2d92ee-b859-4168-b649-4fc681310015"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.460012997Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.464139265Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.492657376Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"188c0cc2-610e-47bb-9e91-c64b7c3d3f29","remote_addr":"[fd00:1122:3344:101::1]:64353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38769244,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.604659983Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1322"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.608387711Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.608819352Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.637073136Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:707"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.642611951Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.743535207Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.823406559Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:37.911029624Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:38.027319885Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:38.184666625Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:38.331086058Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:38.513454466Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:38.679887236Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:39.104414677Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:39.393141894Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:39.789518654Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:40.223838022Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:40.728521532Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:41.458558181Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:41.809859916Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:41.828081545Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:42.775977717Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:39:43.15195378Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:43.510361131Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:43.814513524Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:39:44.156453402Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:39:45.159556977Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:45.294513568Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":40,"time":"2024-10-10T05:39:45.812913949Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:39:46.161116481Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:46.692666138Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/bb59b8a8-537f-4e8d-8b20-1c885a53f692)","v":0,"name":"SledAgent","level":50,"time":"2024-10-10T05:39:47.163704032Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:49.950937843Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12752043-80a8-4a2c-b760-6ab5dd859dbc\", \"content-length\": \"173\", \"date\": \"Thu, 10 Oct 2024 05:39:50 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"6990b2e2-c3e9-4ea3-84d2-9103f8c16a9b\\\"\", request_id: \"12752043-80a8-4a2c-b760-6ab5dd859dbc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:50.478193207Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.228451712Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.230876016Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3600d3ee-db7b-4756-acf7-cccfc42e4b91","remote_addr":"[fd00:1122:3344:101::b]:57751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1698,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.341910488Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.342093413Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"12ee45ef-7a05-458c-a731-2ca74a3832e5","remote_addr":"[fd00:1122:3344:101::c]:37173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.40966246Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:35813"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.412540576Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.663211952Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"269038c2-ec29-4d83-b2c0-7beb301b0d97","remote_addr":"[fd00:1122:3344:101::c]:35813","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":253372,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.663476079Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58040"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.663586762Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50167"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.663648544Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49854"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.663724276Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:32795"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.663768257Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48082"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.665523223Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.666326564Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"b99d3b42-28b3-4d1a-b2c3-4f2318eb4cfe","remote_addr":"[fd00:1122:3344:101::a]:43310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":442,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.66654703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.666660763Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.666717835Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.666727145Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.666950101Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3d8c29fd-8017-4fb8-a480-155ae4ec0479","remote_addr":"[fd00:1122:3344:101::c]:37173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3450,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.666973561Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"23daa84c-da1d-45a6-b24b-d51b29bc8bff","remote_addr":"[fd00:1122:3344:101::b]:57751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3449,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.667086724Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4e7b59af-8d59-4c2b-bbdc-23c89da43683","remote_addr":"[fd00:1122:3344:101::b]:50167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2658,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.667593147Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.667608398Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cb74ad1b-c0db-463f-aa2b-66e2eeea7b1b","remote_addr":"[fd00:1122:3344:101::a]:49854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1098,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.667745091Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.674057908Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"7d9542de-f943-4957-9522-cc004c89cc55","remote_addr":"[fd00:1122:3344:101::c]:32795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7601,"response_code":"200"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.678511325Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.68097676Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"9e3face6-88bf-4ef0-aa79-d2def8b479b8","remote_addr":"[fd00:1122:3344:101::c]:35813","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":17086,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.71520998Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.715303053Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"78430575-5206-4a77-b18e-6fb0a0fe4246","remote_addr":"[fd00:1122:3344:101::a]:49854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":263,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.796501339Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"179b7280-8abe-4dc5-90f5-2678502d27fb","remote_addr":"[fd00:1122:3344:101::b]:57751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":17310,"response_code":"204"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.838588327Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:469","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.83909432Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d52614e7-c9bc-4e4f-a03c-7b655f0630e0","remote_addr":"[fd00:1122:3344:101::a]:49854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5111,"response_code":"204"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.843917397Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:469","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.843970248Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"33b5e24e-a8bb-4d84-8a30-f8bda59b40c4","remote_addr":"[fd00:1122:3344:101::c]:37173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":789,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.952539976Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"37d9edf9-00da-4017-aa9d-042cbaaa73df","remote_addr":"[fd00:1122:3344:101::a]:48082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":285991,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.954776384Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"4d9758cc-4e9f-45ea-9a4d-a1d43a2f4f24","remote_addr":"[fd00:1122:3344:101::b]:58040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":287748,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.955033611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.955078192Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.955330619Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.955554535Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"1b674686-8d15-4fed-be34-8ea86d530d81","remote_addr":"[fd00:1122:3344:101::a]:48082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":576,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.956045058Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.956081569Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.956301975Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:52.957082315Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"46381bc4-2d8b-4e92-8a27-051a56300bb6","remote_addr":"[fd00:1122:3344:101::b]:58040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1095,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.202589835Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42848"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.202697818Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.443502504Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"84484153-8185-48fd-9fae-eb30d6506aa1","remote_addr":"[fd00:1122:3344:101::c]:42848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":240774,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.443983837Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.444052909Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.444467389Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:53.444775508Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"f9bc65d4-c8f9-4cd9-a813-09635952edf6","remote_addr":"[fd00:1122:3344:101::c]:42848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":908,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.126775593Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60612"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128253862Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 454da6a4-8739-4b6e-be7d-17732fe9c6e8, kind: Service { id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6 }, name: Name(\"external-dns-74dd5392-71d1-4b7d-8d5d-310d36142bb6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 140])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 69c8d488-1779-4220-a08e-6e85af04c2ff, kind: Service { id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d }, name: Name(\"external-dns-1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 182])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 631bf7fb-60fd-452f-b2ea-575e055c74ab, kind: Service { id: 58050463-80d5-4fd3-a093-6540c06ff09f }, name: Name(\"nexus-58050463-80d5-4fd3-a093-6540c06ff09f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 207])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6847cb1c-f166-44b9-b911-23c39ad27482, kind: Service { id: db838d4d-3635-4797-be9c-a0ea76b02d91 }, name: Name(\"nexus-db838d4d-3635-4797-be9c-a0ea76b02d91\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 204])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90034877-22f9-4939-9b59-843b55262729, kind: Service { id: 9064e1da-7499-479b-93ca-5ae4b7134452 }, name: Name(\"nexus-9064e1da-7499-479b-93ca-5ae4b7134452\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128454307Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128479078Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128496848Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128517659Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128534179Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.128544029Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.12856925Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"43b505a0-496f-48b0-9c60-da6f6cfebd91","remote_addr":"[fd00:1122:3344:101::c]:60612","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1743,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.129297679Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.367495527Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62923"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.367779185Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.574549656Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"d6613481-9c0b-4283-a200-3d1450f93810","remote_addr":"[fd00:1122:3344:101::c]:62923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":206765,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.575081019Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.575151071Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.57584111Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:54.576117597Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"3fb2ca56-7f2c-48f2-ab71-0c10d0a9b650","remote_addr":"[fd00:1122:3344:101::c]:62923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.882297377Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64449"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883218241Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 454da6a4-8739-4b6e-be7d-17732fe9c6e8, kind: Service { id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6 }, name: Name(\"external-dns-74dd5392-71d1-4b7d-8d5d-310d36142bb6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 140])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 69c8d488-1779-4220-a08e-6e85af04c2ff, kind: Service { id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d }, name: Name(\"external-dns-1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 182])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 631bf7fb-60fd-452f-b2ea-575e055c74ab, kind: Service { id: 58050463-80d5-4fd3-a093-6540c06ff09f }, name: Name(\"nexus-58050463-80d5-4fd3-a093-6540c06ff09f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 207])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6847cb1c-f166-44b9-b911-23c39ad27482, kind: Service { id: db838d4d-3635-4797-be9c-a0ea76b02d91 }, name: Name(\"nexus-db838d4d-3635-4797-be9c-a0ea76b02d91\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 204])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90034877-22f9-4939-9b59-843b55262729, kind: Service { id: 9064e1da-7499-479b-93ca-5ae4b7134452 }, name: Name(\"nexus-9064e1da-7499-479b-93ca-5ae4b7134452\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883269093Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883291563Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883304283Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883319574Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883335254Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883357735Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:55.883382555Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"62b413f7-71da-4032-94c3-acf2656b6dd6","remote_addr":"[fd00:1122:3344:101::a]:64449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1008,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.043880759Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38435"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044083014Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 454da6a4-8739-4b6e-be7d-17732fe9c6e8, kind: Service { id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6 }, name: Name(\"external-dns-74dd5392-71d1-4b7d-8d5d-310d36142bb6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 140])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 69c8d488-1779-4220-a08e-6e85af04c2ff, kind: Service { id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d }, name: Name(\"external-dns-1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 182])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 631bf7fb-60fd-452f-b2ea-575e055c74ab, kind: Service { id: 58050463-80d5-4fd3-a093-6540c06ff09f }, name: Name(\"nexus-58050463-80d5-4fd3-a093-6540c06ff09f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 207])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6847cb1c-f166-44b9-b911-23c39ad27482, kind: Service { id: db838d4d-3635-4797-be9c-a0ea76b02d91 }, name: Name(\"nexus-db838d4d-3635-4797-be9c-a0ea76b02d91\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 204])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90034877-22f9-4939-9b59-843b55262729, kind: Service { id: 9064e1da-7499-479b-93ca-5ae4b7134452 }, name: Name(\"nexus-9064e1da-7499-479b-93ca-5ae4b7134452\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044139866Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044177167Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044194277Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044208697Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044239128Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044250329Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.044269649Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"71776bfe-21fd-4921-bde9-d09951f14e6f","remote_addr":"[fd00:1122:3344:101::b]:38435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":330,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.066991187Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33243"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067214113Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 454da6a4-8739-4b6e-be7d-17732fe9c6e8, kind: Service { id: 74dd5392-71d1-4b7d-8d5d-310d36142bb6 }, name: Name(\"external-dns-74dd5392-71d1-4b7d-8d5d-310d36142bb6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 140])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 69c8d488-1779-4220-a08e-6e85af04c2ff, kind: Service { id: 1dde945c-0d24-4fa1-bc15-9f5ffae3c10d }, name: Name(\"external-dns-1dde945c-0d24-4fa1-bc15-9f5ffae3c10d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 182])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 631bf7fb-60fd-452f-b2ea-575e055c74ab, kind: Service { id: 58050463-80d5-4fd3-a093-6540c06ff09f }, name: Name(\"nexus-58050463-80d5-4fd3-a093-6540c06ff09f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 207])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6847cb1c-f166-44b9-b911-23c39ad27482, kind: Service { id: db838d4d-3635-4797-be9c-a0ea76b02d91 }, name: Name(\"nexus-db838d4d-3635-4797-be9c-a0ea76b02d91\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 204])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90034877-22f9-4939-9b59-843b55262729, kind: Service { id: 9064e1da-7499-479b-93ca-5ae4b7134452 }, name: Name(\"nexus-9064e1da-7499-479b-93ca-5ae4b7134452\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067264184Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067287164Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067323116Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067344536Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067357077Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067367187Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:56.067389457Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"345c3d5d-43b4-47e9-9198-93c536511d08","remote_addr":"[fd00:1122:3344:101::c]:33243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":285,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:58.980899791Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:39:58.981407304Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"bde48c3c-8a2b-4d5b-b77e-b77b5c694068","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39589","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":55,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:03.535422555Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:03.535534638Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b6b80a94-b63f-40a6-834f-1745fec9d5cd","remote_addr":"[fd00:1122:3344:101::c]:62834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:03.550569804Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:03.550588165Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"dca53116-3b3e-4d41-b7e6-cb5b13e75e28","remote_addr":"[fd00:1122:3344:101::c]:62834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:03.560236059Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"aa02aaca-c515-4ad9-a242-d6ac5ff300f0","remote_addr":"[fd00:1122:3344:101::c]:62834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.001624861Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.001777425Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e543361a-3a7e-43a2-9668-575ec73fdc3e","remote_addr":"[fd00:1122:3344:101::b]:61385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.024314858Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.024336749Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"27bc3474-6bb3-4344-b74d-1f42c184694f","remote_addr":"[fd00:1122:3344:101::b]:61385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.036435557Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0cdbd9fd-d762-40a8-8061-30db5c37f468","remote_addr":"[fd00:1122:3344:101::b]:61385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1015,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.148844805Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44464"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.14904329Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"59e42326-9f79-4e95-9325-3ea4f9e5ef65","remote_addr":"[fd00:1122:3344:101::b]:44464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.258622033Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.258762867Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3d3e4663-faab-4e72-ab14-aba7f0d28a85","remote_addr":"[fd00:1122:3344:101::c]:61107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.278347262Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.278374303Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8f31121d-05fc-4f5e-92bc-ab78c2fb21be","remote_addr":"[fd00:1122:3344:101::c]:61107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.307769366Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55034"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.308265479Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.397378114Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.460839184Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3771407c-f6e8-4c69-9a50-366298be406e","remote_addr":"[fd00:1122:3344:101::b]:55034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":152963,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.461135702Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55511"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.461208634Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47300"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.461275736Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.461369678Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a6d92dfe-e912-443e-aa90-97707a2d2d93","remote_addr":"[fd00:1122:3344:101::c]:61107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":334,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.462002015Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"498293ac-901d-4c3c-af29-b8a0c0b2b3e1","remote_addr":"[fd00:1122:3344:101::b]:55034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":815,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.462547069Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a46081cb-5b60-48cf-8b45-01f2a00ce2ca","remote_addr":"[fd00:1122:3344:101::a]:55511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":663,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.46256636Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.468498186Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3fe561b8-6ba0-4fd8-a8db-253d35da8fc7","remote_addr":"[fd00:1122:3344:101::c]:47300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6917,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.503956319Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46973"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.504480203Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"5614452d-2784-4f6a-87d2-29e896886c14","remote_addr":"[fd00:1122:3344:101::a]:46973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.508451697Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.523311948Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/6288dae8-9661-43f6-860a-cae2ca73d224/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.524087528Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.527923599Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.527987251Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2ca04144-7be2-43d0-9d17-164ef5a4b0dc","remote_addr":"[fd00:1122:3344:101::a]:55511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":764,"response_code":"204"} {"msg":"Writing ledger to /pool/int/75bdb1b2-c020-4f15-b179-3702eca17305/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.528913075Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.530055596Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.530130727Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"f945bb1e-9e0e-4fd5-a70e-d476e5cddf16","remote_addr":"[fd00:1122:3344:101::b]:55034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9585,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.541065585Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f19d978f-e786-49d9-ba58-fc592ca312d2","remote_addr":"[fd00:1122:3344:101::a]:55511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1007,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.571501746Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2f1b2b56-790e-44e5-bbc0-e442d889b770","remote_addr":"[fd00:1122:3344:101::c]:54138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110016,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.572097642Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"95515223-05c1-4792-9d32-c5adeedf1c82","remote_addr":"[fd00:1122:3344:101::c]:54138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":338,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.657302474Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33866"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.657455258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.689582933Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.745483294Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"730d1db4-5445-45ee-9b48-f85901898bb8","remote_addr":"[fd00:1122:3344:101::a]:33866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88083,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:22.745857604Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8ed82e48-ca17-444e-a187-b999ee5f437f","remote_addr":"[fd00:1122:3344:101::a]:33866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:29.015689464Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42898"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:29.015893879Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"1ce3768e-5664-4bf3-83bb-c8ef2a735aad","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42898","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:51.178051086Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.049772443Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.050973095Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"9fe8c32a-9260-46bf-b412-f59abd75c1e5","remote_addr":"[fd00:1122:3344:101::b]:57201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1086,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.096282957Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53850"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.096422681Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"305646d0-1bda-4664-817b-104c585c7980","remote_addr":"[fd00:1122:3344:101::b]:53850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.168029275Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.168076867Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f436d8d3-0c77-49a8-a208-8d62ac06bf7f","remote_addr":"[fd00:1122:3344:101::b]:53850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.179014874Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2fb00cdf-4521-4b7a-92ba-f1e438f4d4b2","remote_addr":"[fd00:1122:3344:101::b]:53850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.540927287Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62002"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.540999069Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d2a64beb-93ac-4914-a07a-19415e1e4001","remote_addr":"[fd00:1122:3344:101::a]:62002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.552746948Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.552959894Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2d05e679-d5f5-4a3a-bb42-e0d2bd5f338d","remote_addr":"[fd00:1122:3344:101::c]:53414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.55319658Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.553270442Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1b0a34e9-91b0-4196-9dcf-4d802e48df9e","remote_addr":"[fd00:1122:3344:101::c]:47688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":22,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.557065372Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38278"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.557173255Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.60257877Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.663730579Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"99a825de-8790-4f28-949b-ec6ea8466cff","remote_addr":"[fd00:1122:3344:101::b]:38278","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106601,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.663964815Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.663983816Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f7020694-0c72-4c97-9e61-0f81aae0e633","remote_addr":"[fd00:1122:3344:101::c]:47688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.665121765Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.665138776Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5139d553-6187-4f5a-8d19-30ba6c3ae037","remote_addr":"[fd00:1122:3344:101::a]:62002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.665256659Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4682bca5-3738-4b03-ab8d-db15716472fa","remote_addr":"[fd00:1122:3344:101::b]:38278","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.676403592Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"bfbfdfe3-b12d-4665-a122-eaaa0f9671e3","remote_addr":"[fd00:1122:3344:101::c]:47688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":198,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.766967995Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54404"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.767069578Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.806467755Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.864013629Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"065667f8-e8c3-45a7-88ec-9d22e201607b","remote_addr":"[fd00:1122:3344:101::c]:54404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96989,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.864338277Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34230"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.864650336Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cc9c71b0-0de2-4e33-8f57-040f69ec1a9e","remote_addr":"[fd00:1122:3344:101::c]:54404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":273,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.864705947Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"eeb5c5c5-8ff1-46f9-a406-89cf080a45c7","remote_addr":"[fd00:1122:3344:101::a]:62002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":445,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:52.864753708Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2a53f402-3640-4dd8-badd-c4236e43de43","remote_addr":"[fd00:1122:3344:101::a]:34230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":257,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:53.078946904Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33298"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:53.079066578Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:53.121264788Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:53.18291374Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f0875e9e-044e-46dc-a94a-3120016f5ed6","remote_addr":"[fd00:1122:3344:101::a]:33298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103890,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:53.185365814Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f5d97940-fc9b-49af-a93f-39f8827b9630","remote_addr":"[fd00:1122:3344:101::a]:33298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:59.049475298Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:40:59.049589971Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"067d18f0-c372-4792-a52e-9cad24138638","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48439","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.052660245Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.052789038Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ee445a57-d16f-4047-a9b3-ed3981d1e5ac","remote_addr":"[fd00:1122:3344:101::b]:57425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.118739104Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.118767345Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"31eff3a0-37f7-4602-b70e-0b10a3a7aca6","remote_addr":"[fd00:1122:3344:101::b]:57425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.125875301Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44667"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.126064366Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2f57bbcc-9d92-49b2-82a4-66be4d9dd295","remote_addr":"[fd00:1122:3344:101::b]:44667","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.129080356Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1ec6f8bb-5363-4454-a9f4-6917533cd312","remote_addr":"[fd00:1122:3344:101::b]:57425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":156,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.176491823Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56717"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.176590026Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.224192098Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.308506627Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f8bdda97-d969-4355-af7f-3edcec14fb48","remote_addr":"[fd00:1122:3344:101::b]:56717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131943,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.308656261Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47816"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.308803425Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"860e3f71-0223-4c3e-9e9e-e6fa365af1a4","remote_addr":"[fd00:1122:3344:101::c]:47816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.31013707Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5efa9e3a-4245-420a-8277-3184c6abb4d2","remote_addr":"[fd00:1122:3344:101::b]:56717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1339,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.360272969Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.36030005Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cc931233-3e46-468f-b1b9-6665108bb61c","remote_addr":"[fd00:1122:3344:101::c]:47816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.371398392Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"47ebc4a5-f8d3-49cd-873f-61f727a71ff6","remote_addr":"[fd00:1122:3344:101::c]:47816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":236,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.399909502Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38911"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.400019805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.442549464Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.508069438Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2ff9f353-3a47-48ed-a5ae-d9b07de6c36b","remote_addr":"[fd00:1122:3344:101::c]:38911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108097,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.508354566Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44697"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.508415487Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63122"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.508706625Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"20722ee1-99e7-47f5-952d-697b289eb4ba","remote_addr":"[fd00:1122:3344:101::a]:44697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.508837708Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c966456e-755b-4d1c-a4fd-59a42a35c54b","remote_addr":"[fd00:1122:3344:101::c]:38911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":356,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.509415924Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"29b7aca8-6863-491b-936e-bce2a34727e7","remote_addr":"[fd00:1122:3344:101::c]:63122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.523982637Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.524048079Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4f6ebfe-6623-429c-b4f9-b87d5a0c6d5b","remote_addr":"[fd00:1122:3344:101::a]:44697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.539436954Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49574"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.539542096Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.581884751Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.640038291Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8dd4987e-bf48-49ee-8dab-278443d40a35","remote_addr":"[fd00:1122:3344:101::a]:49574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100538,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.640210285Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60246"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.640462472Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d960290b-db77-447c-bb2c-1b898db8947f","remote_addr":"[fd00:1122:3344:101::a]:49574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.641467438Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d2048ae2-2200-4f00-a4a5-9bc8b5c8f928","remote_addr":"[fd00:1122:3344:101::a]:60246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1077,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:22.641650743Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9971d96e-8bf5-4653-9c00-857e9f6504f0","remote_addr":"[fd00:1122:3344:101::a]:44697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1330,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:23.141937107Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:59625"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:23.281990202Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"d467aea3-b17d-426a-a5fc-71c4786b0236","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":138669,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:29.105816046Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:29.10595671Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"7067b1bd-251a-4e1e-9d01-33cd04dfed70","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43563","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.212907288Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.213247827Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d07a0472-04a3-40d5-8e8a-f78e2e165d72","remote_addr":"[fd00:1122:3344:101::b]:55544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.28976889Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.289806201Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"459724a3-76f6-4e17-82f4-0845fe839551","remote_addr":"[fd00:1122:3344:101::b]:55544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.300681437Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e973a80d-2e03-472e-b895-4801ee7529d0","remote_addr":"[fd00:1122:3344:101::b]:55544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":241,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.315759534Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61504"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.316563505Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e6a7f24b-a0ed-4421-9990-c6a1dade4ee9","remote_addr":"[fd00:1122:3344:101::b]:61504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.337765903Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.337860026Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"dc4ce495-977e-4456-8cf9-266ef93924dc","remote_addr":"[fd00:1122:3344:101::c]:60264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.412470309Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57778"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.412610793Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.456667882Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.529524729Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d9f14015-7cd5-4df4-98e5-ebade689bb8f","remote_addr":"[fd00:1122:3344:101::b]:57778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116950,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.529868208Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.531109141Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"53293e3c-a189-41c1-9c05-8be8497d29ac","remote_addr":"[fd00:1122:3344:101::a]:38266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1095,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.531141121Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.531159172Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6bbd6107-d704-4b52-9745-99af0c0792cf","remote_addr":"[fd00:1122:3344:101::c]:60264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1260,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.531236744Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"36b05478-0c46-4163-9333-6d5f91dc111e","remote_addr":"[fd00:1122:3344:101::b]:57778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1315,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.55045061Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:32940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.552214456Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4d995baa-9be2-43ef-ad9e-41a66cd73683","remote_addr":"[fd00:1122:3344:101::c]:32940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1692,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.559301162Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a2e0a99a-bf9f-40a5-90ce-1572e093ee01","remote_addr":"[fd00:1122:3344:101::c]:60264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":308,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.583112789Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.58313237Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ede8e0c3-2cb3-4291-ab27-c30e83b9b4fc","remote_addr":"[fd00:1122:3344:101::a]:38266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.592565328Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b506eb8f-90a7-4364-8edc-d5447baad558","remote_addr":"[fd00:1122:3344:101::a]:38266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":254,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.660321851Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34248"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.661281386Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.703487297Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.765367685Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d85d132c-dc38-4a1b-b180-048567958f9c","remote_addr":"[fd00:1122:3344:101::c]:34248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104980,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.765729364Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d459830b-3dc1-48b6-9364-2d9a03d3f5ac","remote_addr":"[fd00:1122:3344:101::c]:34248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.773599552Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:52.773878939Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3d5359df-949e-4c5a-995f-59af59476a47","remote_addr":"[fd00:1122:3344:101::a]:62194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:53.013807372Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48099"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:53.015128757Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:53.055742976Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:53.11251653Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"221119b6-2af5-4edf-a43d-7009e6116396","remote_addr":"[fd00:1122:3344:101::a]:48099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98648,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:53.112776417Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"92509e2c-a3dc-4f17-9456-5a8d5fe6ae85","remote_addr":"[fd00:1122:3344:101::a]:48099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:53.150767436Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"74b82c0a-8d81-41f2-bb19-4940a1d5bda4","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9236,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:59.144274414Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40856"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:41:59.144533661Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"21246375-3ad1-471a-9378-d07ce1ae1f27","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40856","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.04363328Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.043749943Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1bdc4d7d-6d15-44a8-88d1-1d32b951c16c","remote_addr":"[fd00:1122:3344:101::b]:57977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.097789975Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.097847127Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cf3b1f19-bc78-4bed-983c-41076b0e75f7","remote_addr":"[fd00:1122:3344:101::b]:57977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.107491621Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d904c135-7c24-4fbb-8239-acf4c935f2c3","remote_addr":"[fd00:1122:3344:101::b]:57977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":335,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.208533629Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:33669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.210181902Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"be59cdf3-4928-4d7e-9462-1425bad9f357","remote_addr":"[fd00:1122:3344:101::b]:33669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1569,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.231511864Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59875"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.231605156Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.283298456Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.348079411Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"526014bb-df39-4a15-b60c-dab9366f57e9","remote_addr":"[fd00:1122:3344:101::b]:59875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116513,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.348219124Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.348364218Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"423b238d-3c98-493f-8761-d47693cb6536","remote_addr":"[fd00:1122:3344:101::c]:60742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.349350994Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"71508d0b-d1cb-4f62-8101-b76354a5663c","remote_addr":"[fd00:1122:3344:101::b]:59875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":925,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.396805083Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.396829184Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"78c0f88f-3e16-40da-a57e-0187ab83544f","remote_addr":"[fd00:1122:3344:101::c]:60742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.401381474Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.401512227Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"85b5c973-3ea7-4d6c-a7e1-e572eed101d1","remote_addr":"[fd00:1122:3344:101::a]:54493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":56,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.412994919Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33943"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.413182874Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"1f0bc9b4-90cf-4b82-a811-c3ce93922360","remote_addr":"[fd00:1122:3344:101::c]:33943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.416662085Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"fd297ca3-c324-4c84-a325-89fd7982ac1a","remote_addr":"[fd00:1122:3344:101::c]:60742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":246,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.450001793Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.450032504Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"17c10630-a39e-4ea6-9edc-1680d5a358f3","remote_addr":"[fd00:1122:3344:101::a]:54493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.458007233Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40797"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.458106286Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.505064202Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.566343664Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0f71dcf6-203c-4de3-b3d0-7dedebb9320b","remote_addr":"[fd00:1122:3344:101::c]:40797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108276,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.566720474Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51749"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.566810927Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46572"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.566969921Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4c517108-f655-4311-8876-a41ebe94ce57","remote_addr":"[fd00:1122:3344:101::c]:40797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":256,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.567100904Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7524a61f-8b6f-40b8-9b43-2e0aa3007235","remote_addr":"[fd00:1122:3344:101::a]:54493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":414,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.567963327Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.568042249Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"6d5f86da-9705-4764-b1ef-895e0a18568c","remote_addr":"[fd00:1122:3344:101::a]:46572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1035,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.609161241Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.660004199Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7f7370a5-500f-402c-be3f-0f237de583f6","remote_addr":"[fd00:1122:3344:101::a]:51749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92926,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:22.660318907Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cfb48226-93ca-4caa-9ab1-48e22d926d4e","remote_addr":"[fd00:1122:3344:101::a]:51749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:23.148495242Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"31d6f757-35c1-46aa-af36-c3e66ac4e922","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7725,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:29.178097591Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:29.178257305Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"6220b1c3-ac0f-400a-a7e1-005fe1d4a694","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41590","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":55,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.210985509Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.211144224Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a611e388-4e41-4409-a769-772006f13cb4","remote_addr":"[fd00:1122:3344:101::b]:47188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.308242919Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.30828364Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8a2ffa9d-12c7-4037-851a-b2aa0eb01f03","remote_addr":"[fd00:1122:3344:101::b]:47188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.32654048Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c6477ffd-95a7-47db-bbcc-c408db4bcae4","remote_addr":"[fd00:1122:3344:101::b]:47188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":285,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.494536021Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53766"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.494855999Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e8dc7f1f-e32d-4269-b291-422794378c04","remote_addr":"[fd00:1122:3344:101::b]:53766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.530216009Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58723"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.530361953Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.575522581Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.645501263Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f1e41895-fd4b-4054-84b9-fc48398edb4f","remote_addr":"[fd00:1122:3344:101::b]:58723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.64578471Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40477"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.645876983Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.650615957Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"df27abc2-8811-4a93-b502-b447d26178df","remote_addr":"[fd00:1122:3344:101::a]:63801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":467,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.650759951Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6ab991ee-1444-4f6c-8b65-aaff5cb19bba","remote_addr":"[fd00:1122:3344:101::c]:40477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":506,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.650796722Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"37363fa4-9d49-409b-8c02-3c1103aed94c","remote_addr":"[fd00:1122:3344:101::b]:58723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4889,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.666624079Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.666642629Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"87fe77b6-7b69-4463-b22d-3199c3e267fb","remote_addr":"[fd00:1122:3344:101::c]:40477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":68,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.746004827Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.746173372Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"70499816-8ad6-4af7-a317-5705ce1e5efd","remote_addr":"[fd00:1122:3344:101::c]:44357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":122,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.750119966Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"86187e78-8f83-43f7-936c-dedd2d759208","remote_addr":"[fd00:1122:3344:101::c]:40477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":249,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.760631892Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47564"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.760763666Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0d25d018-a48e-4934-a333-db72bd29b1cc","remote_addr":"[fd00:1122:3344:101::a]:47564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.808481871Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.808510872Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b0a6eb20-8dde-4618-b572-924ddc918326","remote_addr":"[fd00:1122:3344:101::a]:63801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.817488568Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"30d2161c-1adc-4477-8bbd-fadb955d0296","remote_addr":"[fd00:1122:3344:101::a]:63801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":844,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.825412987Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53246"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.825510859Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.86656232Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.925886721Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"03cbfc1d-e470-4c37-8d0b-354c949932c1","remote_addr":"[fd00:1122:3344:101::a]:53246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100415,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.926038105Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34965"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.926187349Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.92624394Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"92d90e59-79ed-4776-bfaf-4de015446aab","remote_addr":"[fd00:1122:3344:101::a]:53246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":187,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:52.964273251Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:53.026792286Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"42ce166d-63d4-481d-983b-071569b337aa","remote_addr":"[fd00:1122:3344:101::c]:34965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100659,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:53.0280647Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9a42054f-57e4-403b-9a96-3a14970ad3ff","remote_addr":"[fd00:1122:3344:101::c]:34965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1021,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:53.149153295Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"273e3ca5-88e8-45da-854f-3cbe40a9f048","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8941,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:59.210360665Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:42:59.210493668Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"51d997de-706e-41c9-bc2a-cd6c35594c10","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62226","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.076480146Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.07660285Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7b46a134-bdd2-4e5e-ac01-f0423152b991","remote_addr":"[fd00:1122:3344:101::b]:40343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.13057668Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.130611961Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4e4d6085-a7a2-45f3-b85f-0e013f7598d8","remote_addr":"[fd00:1122:3344:101::b]:40343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.143839559Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6cff5883-aea8-41b8-84f9-b81cdcaa607c","remote_addr":"[fd00:1122:3344:101::b]:40343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":994,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.206193619Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61312"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.206401365Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"96169e25-2f2f-42c2-9724-0e3ddeba6223","remote_addr":"[fd00:1122:3344:101::b]:61312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.222267993Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57451"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.222362745Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.274513777Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.346096711Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8c0753a9-81d6-44ed-b5d1-02dfd39dcb01","remote_addr":"[fd00:1122:3344:101::b]:57451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123764,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.346347067Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53229"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.347682142Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b02c3e4b-4a49-47e5-9887-3ab9a9bdd5a6","remote_addr":"[fd00:1122:3344:101::c]:53229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1161,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.347758304Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e94436e1-8397-4292-82f1-0a6f2d2338de","remote_addr":"[fd00:1122:3344:101::b]:57451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1386,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.409808827Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.409943491Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8ba3e95f-3dcc-44df-80fd-5786ea50b799","remote_addr":"[fd00:1122:3344:101::a]:59227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.423642251Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.423668862Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3b98b156-bff9-400c-b885-af749edf2342","remote_addr":"[fd00:1122:3344:101::c]:53229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.460753638Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.460775829Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b7b5dcee-fd33-4c0b-a44c-435f4537fa28","remote_addr":"[fd00:1122:3344:101::a]:59227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":55,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.490181352Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.490363427Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"c3c218b6-b9fc-432f-9f06-91d56c3f958e","remote_addr":"[fd00:1122:3344:101::a]:37805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.498030829Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d915f505-033b-475c-bf17-5b60ef401c99","remote_addr":"[fd00:1122:3344:101::a]:59227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":256,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.520150981Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43864"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.520250143Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.56574318Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.627736772Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"4d964c22-a16b-4dbf-9366-b3d86837f632","remote_addr":"[fd00:1122:3344:101::a]:43864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107518,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.627879875Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40880"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.627913536Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62028"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.628144722Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.63223153Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ab086438-7c54-4720-856c-b6848de4eedb","remote_addr":"[fd00:1122:3344:101::a]:43864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4197,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.632361793Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2d11226f-cc97-4efd-ac5b-96830be816f3","remote_addr":"[fd00:1122:3344:101::c]:53229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4221,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.632571209Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e4addd17-1d89-4ea3-847a-3e590676b8c7","remote_addr":"[fd00:1122:3344:101::c]:40880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4367,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.665923767Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.724548429Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ec317e72-22f2-44df-ad82-4684e9b1c804","remote_addr":"[fd00:1122:3344:101::c]:62028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96513,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:22.7249556Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4037764a-7282-4d98-8182-46f017a16348","remote_addr":"[fd00:1122:3344:101::c]:62028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":177,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:23.148639178Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"a186b579-c93b-4ee2-8146-d801d704916d","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8466,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:29.246979505Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41443"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:29.247108529Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"692f42f8-a12f-4ac6-ac96-52bf3aa84a82","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41443","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.24706356Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.247276165Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"719a31d8-1d8d-4853-bdfa-54c999e08e12","remote_addr":"[fd00:1122:3344:101::b]:57468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.296173402Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.296267584Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"73a41612-a98b-4d70-bcc5-967b6a51610e","remote_addr":"[fd00:1122:3344:101::b]:56181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.314301219Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.31433673Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"14d97b65-8cbe-4166-b1b6-eb7dd33866e9","remote_addr":"[fd00:1122:3344:101::b]:56181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.324650991Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"99b61a4d-a0e8-4aea-92da-a2e347ff6fcc","remote_addr":"[fd00:1122:3344:101::b]:56181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.381687282Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.381787125Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"55671b90-3787-46e8-83a2-12f63ef3a443","remote_addr":"[fd00:1122:3344:101::c]:55098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.398397542Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.398425743Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"aa977155-f929-4b5c-b6c1-e13729d264e8","remote_addr":"[fd00:1122:3344:101::c]:55098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.412692548Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"218362d6-9abd-4718-9497-4a52f99d34c1","remote_addr":"[fd00:1122:3344:101::c]:55098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":284,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.429937262Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51035"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.43103465Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.473708004Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.537634306Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5a23c358-82d7-4ea9-a714-de2370a2121e","remote_addr":"[fd00:1122:3344:101::b]:51035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107634,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.537756449Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.538012276Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"78b8442b-3446-4bcd-b7d3-a4d8a3a77147","remote_addr":"[fd00:1122:3344:101::c]:47493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.538064007Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e55e677a-868f-417d-8f9e-6491dab7cf1a","remote_addr":"[fd00:1122:3344:101::b]:51035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.629082602Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42696"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.629228436Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3d6bc76c-34b7-4473-b6fc-0f21bbbe853e","remote_addr":"[fd00:1122:3344:101::a]:42696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.662697446Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42126"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.662863861Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.707178187Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.771700625Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f5208cad-90e5-4708-885c-e450e4ba629c","remote_addr":"[fd00:1122:3344:101::c]:42126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108902,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.771978252Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35000"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.772033943Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.772056924Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.772072795Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cf82d54e-3a6b-4d73-9e58-a09831b7e5d2","remote_addr":"[fd00:1122:3344:101::a]:42696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.772353632Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f9f93b1c-2cec-421b-afdb-09ef6c713ce4","remote_addr":"[fd00:1122:3344:101::c]:42126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":292,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.773085561Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d9648dc1-2b98-41c3-bdcd-9f731abccc0e","remote_addr":"[fd00:1122:3344:101::a]:47175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":908,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.773108332Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.783370702Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6e142ada-33f5-473c-aa11-08afa8ca44a6","remote_addr":"[fd00:1122:3344:101::a]:42696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":319,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.822666106Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.883056495Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c5892969-fbf2-4e6f-b97a-0609f33a0535","remote_addr":"[fd00:1122:3344:101::a]:35000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110771,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:52.883397834Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"36022029-f8be-4645-bdbf-a9a8e7374f43","remote_addr":"[fd00:1122:3344:101::a]:35000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:53.312678769Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"04722014-05a7-4c6d-8dbb-75431d15ad01","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8041,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:59.281741854Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48884"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:43:59.281864958Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"097f16b6-ab03-4502-9e57-1b85b3df690c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48884","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.71509959Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.71584463Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0ca6b1ce-a4df-4283-b497-eee34034281c","remote_addr":"[fd00:1122:3344:101::b]:63255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":686,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.732300123Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.732334844Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c0ca1e6c-020a-42b0-a752-fe0d80e83c0b","remote_addr":"[fd00:1122:3344:101::b]:63255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.744490313Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6939ea93-b76b-414f-8e9f-d8eacde0576c","remote_addr":"[fd00:1122:3344:101::b]:63255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1049,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.845888141Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.846055756Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3323dbb4-577f-43e2-8f62-9f7ffdc8323c","remote_addr":"[fd00:1122:3344:101::b]:46621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.982870376Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50779"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:21.983069821Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"8920dd22-91fe-4cb7-8ced-b996adbd2b1c","remote_addr":"[fd00:1122:3344:101::c]:50779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.014322533Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45832"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.014419876Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.057421658Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.117754325Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d4897ccc-66e2-4a26-b5b0-ba15608aff09","remote_addr":"[fd00:1122:3344:101::b]:45832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103372,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.117860238Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33494"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.117903509Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41296"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.118149925Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.122409717Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fdda9f40-30b8-48ff-98e3-97d8b4cb78c6","remote_addr":"[fd00:1122:3344:101::b]:45832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4264,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.122547931Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6fdbcd8f-4742-4618-9b3d-9b09b8417af5","remote_addr":"[fd00:1122:3344:101::c]:33494","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4355,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.15632061Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46869"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.156533975Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b56b50e2-05b1-46e9-9f75-d79b1ed53575","remote_addr":"[fd00:1122:3344:101::a]:46869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.157989454Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.176664865Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.176749157Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6abc87d6-6314-4717-9e32-f270be907ac4","remote_addr":"[fd00:1122:3344:101::c]:33494","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":695,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.191168027Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"688006d5-280f-4a6a-911f-110017613231","remote_addr":"[fd00:1122:3344:101::c]:33494","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1521,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.204047626Z","hostname":"sock","pid":1572,"sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.204121148Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2125d1d6-00a5-45e3-aafb-2815f4c14980","remote_addr":"[fd00:1122:3344:101::a]:46869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":723,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.218731152Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a2defc0b-0a12-461b-9d55-ff1133d5c90b","remote_addr":"[fd00:1122:3344:101::a]:46869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.224149155Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64462"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.227479072Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.227498243Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"892a0f02-3685-4a01-8bf6-719926e9c8b8","remote_addr":"[fd00:1122:3344:101::c]:41296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109455,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.232954096Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"55ff6658-528b-4cf6-acdd-944ba3aa6d23","remote_addr":"[fd00:1122:3344:101::c]:41296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":509,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.267192877Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.323880719Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52511"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.324185207Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"bb76a11c-748d-4382-9f63-60463ceac33e","remote_addr":"[fd00:1122:3344:101::a]:52511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":199,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.325996314Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"599d9bc7-70a6-417f-a1b5-ceaac936cc68","remote_addr":"[fd00:1122:3344:101::a]:64462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101793,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.326238651Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e100a959-0091-4814-9c44-7a068d54ca64","remote_addr":"[fd00:1122:3344:101::a]:64462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-10T05:44:22.976128112Z","hostname":"sock","pid":1572,"uri":"/bb59b8a8-537f-4e8d-8b20-1c885a53f692","method":"GET","req_id":"be977ad1-d97c-411d-a157-70a8bc870972","remote_addr":"[fd00:1122:3344:101::d]:59625","local_addr":"[fd00:1122:3344:101::1]:56188","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"bb59b8a8-537f-4e8d-8b20-1c885a53f692","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7765,"response_code":"200"}