[ Aug 29 15:14:13 Enabled. ]
[ Aug 29 15:14:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Aug 29 15:14:13 Method "start" exited with status 0. ]
[ Aug 29 15:14:13 Rereading configuration. ]
[ Aug 29 15:14:13 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.281156273Z","hostname":"sock","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.281312876Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.333355915Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.333395636Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.446799006Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.455368144Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:13.463155046Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.066581001Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.203448501Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.203529373Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.203645805Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.203767288Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.203804018Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.203822269Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206686205Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:14.206693065Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206706905Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206714415Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206731166Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206777747Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206869039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206897629Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.206909259Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.218333643Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.218360724Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.318394442Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.732440399Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.73246915Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.73247483Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.732486011Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.732492291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.732502931Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.732520321Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.741437786Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.741462086Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:14.829441289Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233679234Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233716775Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233750966Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233758776Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233777346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233789556Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233833517Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233846028Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233862918Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233872018Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233877548Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233893798Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233897858Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233901769Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233920399Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233925559Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233930159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233935099Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233939539Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.233943489Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.23396067Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.23396559Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.23397883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.23398329Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.23398789Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.23399291Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234014781Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234020151Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234025901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234030771Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234039061Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234058592Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234062622Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234070062Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234100313Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234112323Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234116733Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234128013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234132933Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234137533Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234171094Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234176654Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234181564Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234185874Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234214375Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234220215Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234239135Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234243885Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234248735Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234253945Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234258025Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234262546Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234294676Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234301466Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234305726Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234316567Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234320877Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234355007Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234359867Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234363828Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234367848Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234372058Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234389218Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234396138Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234405168Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234412108Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234417009Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234446239Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.234451549Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.245029187Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:730"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.245059997Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999922868s","next_cleanup":"Instant { tv_sec: 20773, tv_nsec: 845925025 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.245107778Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.245156599Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.24521924Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.245234611Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248145808Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248229379Z","hostname":"sock","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV2 {\n        rack_subnet: Ipv6Net {\n            addr: fd00:1122:3344:100::,\n            width: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV2 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Net {\n                                addr: 0.0.0.0,\n                                width: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                        vlan_id: None,\n                        local_pref: None,\n                    },\n                ],\n                addresses: [\n                    UplinkAddressConfig {\n                        address: V4(\n                            Ipv4Net {\n                                addr: 10.151.2.174,\n                                width: 24,\n                            },\n                        ),\n                        vlan_id: None,\n                    },\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n                lldp: None,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248449303Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1053"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248455514Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248515225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248560265Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248574556Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248586376Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248611177Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248624177Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248636187Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248640717Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248644477Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248648117Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248660578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.248694638Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249018414Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage: Plan {\n    rack_id: 0c368d89-20a1-4c09-9eea-581bb60b3770,\n    sleds: {\n        [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n            generation: 0,\n            schema_version: 1,\n            body: StartSledAgentRequestBody {\n                id: b9c40332-a231-42db-adf0-b267d663a311,\n                rack_id: 0c368d89-20a1-4c09-9eea-581bb60b3770,\n                use_trust_quorum: false,\n                is_lrtq_learner: false,\n                subnet: Ipv6Subnet {\n                    net: Ipv6Net {\n                        addr: fd00:1122:3344:101::,\n                        width: 64,\n                    },\n                },\n            },\n        },\n    },\n    config: RackInitializeRequest {\n        trust_quorum_peers: None,\n        bootstrap_discovery: OnlyOurs,\n        ntp_servers: [\n            \"0.pool.ntp.org\",\n        ],\n        dns_servers: [\n            1.1.1.1,\n            9.9.9.9,\n        ],\n        internal_services_ip_pool_ranges: [\n            V4(\n                Ipv4Range {\n                    first: 10.151.2.164,\n                    last: 10.151.2.173,\n                },\n            ),\n        ],\n        external_dns_ips: [\n            10.151.2.164,\n            10.151.2.165,\n        ],\n        external_dns_zone_name: \"oxide.test\",\n        external_certificates: \"<redacted>\",\n        recovery_silo: RecoverySiloConfig {\n            silo_name: Name(\n                \"recovery\",\n            ),\n            user_name: UserId(\n                \"recovery\",\n            ),\n            user_password_hash: NewPasswordHash(\n                PasswordHashString {\n                    string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                    encoding: B64,\n                },\n            ),\n        },\n        rack_network_config: RackNetworkConfigV2 {\n            rack_subnet: Ipv6Net {\n                addr: fd00:1122:3344:100::,\n                width: 56,\n            },\n            infra_ip_first: 10.151.2.174,\n            infra_ip_last: 10.151.2.174,\n            ports: [\n                PortConfigV2 {\n                    routes: [\n                        RouteConfig {\n                            destination: V4(\n                                Ipv4Net {\n                                    addr: 0.0.0.0,\n                                    width: 0,\n                                },\n                            ),\n                            nexthop: 10.151.2.100,\n                            vlan_id: None,\n                            local_pref: None,\n                        },\n                    ],\n                    addresses: [\n                        UplinkAddressConfig {\n                            address: V4(\n                                Ipv4Net {\n                                    addr: 10.151.2.174,\n                                    width: 24,\n                                },\n                            ),\n                            vlan_id: None,\n                        },\n                    ],\n                    switch: Switch0,\n                    port: \"qsfp0\",\n                    uplink_port_speed: Speed40G,\n                    uplink_port_fec: None,\n                    bgp_peers: [],\n                    autoneg: false,\n                    lldp: None,\n                },\n            ],\n            bgp: [],\n            bfd: [],\n        },\n        allowed_source_ips: Any,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249225399Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249242429Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1187"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.24927089Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.24928217Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249429203Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249552955Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249605546Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: b9c40332-a231-42db-adf0-b267d663a311, rack_id: 0c368d89-20a1-4c09-9eea-581bb60b3770, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249759239Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:56082","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: b9c40332-a231-42db-adf0-b267d663a311, rack_id: 0c368d89-20a1-4c09-9eea-581bb60b3770, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249853001Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249859471Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249890792Z","hostname":"sock","pid":1562,"file":"sled-agent/src/server.rs:44"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249897672Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249901352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249904812Z","hostname":"sock","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249919992Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249935052Z","hostname":"sock","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249949163Z","hostname":"sock","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249958123Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249962103Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.249967893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.250000204Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.250005654Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.251905761Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4163"}
{"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.251956782Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.282816246Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.304984921Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.322632926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.329321287Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:388"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.329364348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.329374548Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:400"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.329435339Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 140.138782ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.332369697Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dcbaef70-cc44-449f-9aa0-1536ebd05f68\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dcbaef70-cc44-449f-9aa0-1536ebd05f68\" }"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.342579806Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.365333072Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.365376953Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.37033755Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:415"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.37035146Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:127"}
{"msg":"Ensuring dataset oxi_5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.37035811Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:137"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.457910675Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:172"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.47297935Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 167.056344ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.473583482Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85b79dfd-1f9f-4fd6-a68d-b81e28e79b3d\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85b79dfd-1f9f-4fd6-a68d-b81e28e79b3d\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.641298956Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 215.738517ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.641937198Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d904bc70-8fb3-4292-a2c5-9d0e2439db74\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d904bc70-8fb3-4292-a2c5-9d0e2439db74\" }"}
{"msg":"Mounting oxi_5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.67779297Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:180"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.690673633Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.690753604Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.690766914Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.690783435Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:194"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.858177622Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 173.70694ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:15.858879456Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27a347bd-c311-4bb2-99fa-9ebac4737b0a\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27a347bd-c311-4bb2-99fa-9ebac4737b0a\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.033538675Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 192.531334ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.034445903Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef50c20e-a68e-4d4c-ab2e-79fe73fa9b19\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef50c20e-a68e-4d4c-ab2e-79fe73fa9b19\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.228034214Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 193.130365ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.228646606Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11076fbe-32f5-4ffe-b028-ef655211089e\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11076fbe-32f5-4ffe-b028-ef655211089e\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.417052155Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.42242822Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.422464681Z","hostname":"sock","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.422532352Z","hostname":"sock","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:34899","producer_id":"b9c40332-a231-42db-adf0-b267d663a311"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.422897679Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:16.423831438Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"221.993246ms"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.424662524Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.424675104Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.424678864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:914","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:16.424683674Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:16.648282193Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"635.202892ms"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:17.285251235Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.388018331s"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:18.675507747Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.207316312s"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.90857123Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2639"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.908587051Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2857"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.908602451Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2509"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.908607821Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2556"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.908658012Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2938"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.908663412Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.91061418Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2782"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.912248812Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2550"}
{"msg":"Profile for oxz_switch:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgs\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"id\" value='8f88c1d9-b275-491d-b7f9-340e22f40c0e'/>\n        <propval type=\"astring\" name=\"rack_id\" value='0c368d89-20a1-4c09-9eea-581bb60b3770'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/wicketd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12226'/>\n        <propval type=\"astring\" name=\"artifact-address\" value='[fdb0:18c0:4d0d:a02a::2]:12227'/>\n        <propval type=\"astring\" name=\"baseboard-file\" value='/opt/oxide/baseboard.json'/>\n        <propval type=\"astring\" name=\"mgs-address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"nexus-proxy-address\" value='[::]:12229'/>\n        <propval type=\"astring\" name=\"rack-subnet\" value='fd00:1122:3344::'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/switch_zone_setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"gz_local_link_addr\" value='fe80::8:20ff:fe0e:242c'/>\n        <property type=\"astring\" name=\"link_local_links\">\n          <astring_list>\n            <value_node value=\"tfportrear0_0\" />\n            <value_node value=\"oxBootstrap0\" />\n          </astring_list>\n        </property>\n        <propval type=\"astring\" name=\"bootstrap_addr\" value='fdb0:18c0:4d0d:a02a::2'/>\n        <propval type=\"astring\" name=\"bootstrap_vnic\" value='oxBootstrap0'/>\n        <propval type=\"astring\" name=\"baseboard_info\" value='{\"type\":\"pc\",\"identifier\":\"sock\",\"model\":\"i86pc\"}'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/dendrite\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_id\" value='b9c40332-a231-42db-adf0-b267d663a311'/>\n        <propval type=\"astring\" name=\"rack_id\" value='0c368d89-20a1-4c09-9eea-581bb60b3770'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12224'/>\n        <propval type=\"astring\" name=\"front_ports\" value='1'/>\n        <propval type=\"astring\" name=\"rear_ports\" value='1'/>\n        <propval type=\"astring\" name=\"port_config\" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>\n        <propval type=\"astring\" name=\"mgmt\" value='uds'/>\n        <propval type=\"astring\" name=\"uds_path\" value='/opt/softnpu/stuff'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/tfport\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"host\" value='[::1]'/>\n        <propval type=\"astring\" name=\"port\" value='12224'/>\n        <propval type=\"astring\" name=\"flags\" value='--sync-only'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/lldpd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"board_rev\" value='softnpu_front_1_rear_1'/>\n        <propval type=\"astring\" name=\"scrimlet_id\" value='sock'/>\n        <propval type=\"astring\" name=\"scrimlet_model\" value='i86pc'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12230'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/pumpkind\">\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_uuid\" value='b9c40332-a231-42db-adf0-b267d663a311'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='0c368d89-20a1-4c09-9eea-581bb60b3770'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mg-ddm\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"mode\" value='transit'/>\n        <propval type=\"astring\" name=\"dendrite\" value='true'/>\n        <propval type=\"astring\" name=\"sled_uuid\" value='b9c40332-a231-42db-adf0-b267d663a311'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='0c368d89-20a1-4c09-9eea-581bb60b3770'/>\n        <propval type=\"astring\" name=\"interfaces\" value='tfportrear0_0/ll'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/uplink\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.912355995Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:18.912495717Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 67.843637ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:19.165802417Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 94.944248ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:19.357877788Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 242.809621ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:19.532426546Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 380.232042ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:19.881035332Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 640.299106ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:20.468185608Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:20.884988469Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.880013918s"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 683.73984ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:21.214073333Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.115131775s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:22.084354543Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 718.587864ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:23.421786071Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.697949528Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.697997359Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.698028949Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.69804833Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.698314925Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.698508149Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.69856206Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.69857549Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.6985808Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:56082","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.953682655Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.953751907Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.953824658Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.95391273Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.9539185Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.95394032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.953962951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.953977571Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:24.954961211Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.957320105s"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.958378847Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4187","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.958386307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4203"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:24.958395608Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.221939367Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49015"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.222250793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.222403236Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"634692d2-72df-49b4-b5c2-ca8fbbfea75c","remote_addr":"[fd00:1122:3344:101::1]:49015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":321,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.253484785Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36314"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.253611678Z","hostname":"sock","pid":1562,"uri":"/sled-role","method":"GET","req_id":"30678255-9da8-417b-af45-c5b384621f5d","remote_addr":"[fd00:1122:3344:101::1]:36314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":71,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.254300551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.254332402Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.254674968Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.255053076Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:999"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 38bc4c99-f824-42b1-9252-61e07ec12c50, pool_id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: d2db14f3-ed37-4067-aabb-56412ba4cb84, pool_id: 866d818f-82bb-444c-992a-1351dec91efa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 4ebd9cb5-27f4-4c65-94c6-ff793dd0343a, pool_id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 4c637c8b-6729-4597-a52c-8e37a3ad27e4, pool_id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: ce05b51a-cd89-4ebb-b688-efb500dd174e, pool_id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c01d4db6-8364-4378-8e25-1d180755f647, pool_id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 4b4a6b84-2a5f-42e2-9e64-42ad4b741866, pool_id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: ab80f8c3-3ca5-4ec8-99ed-275ba79ffbf5, pool_id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 87286297-b93c-4248-91f6-2951af5888f5, pool_id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.284593344Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:355"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.284705326Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63462"}
{"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285079094Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:856"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 38bc4c99-f824-42b1-9252-61e07ec12c50, pool_id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: d2db14f3-ed37-4067-aabb-56412ba4cb84, pool_id: 866d818f-82bb-444c-992a-1351dec91efa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 4ebd9cb5-27f4-4c65-94c6-ff793dd0343a, pool_id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 4c637c8b-6729-4597-a52c-8e37a3ad27e4, pool_id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: ce05b51a-cd89-4ebb-b688-efb500dd174e, pool_id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c01d4db6-8364-4378-8e25-1d180755f647, pool_id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 4b4a6b84-2a5f-42e2-9e64-42ad4b741866, pool_id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: ab80f8c3-3ca5-4ec8-99ed-275ba79ffbf5, pool_id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 87286297-b93c-4248-91f6-2951af5888f5, pool_id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285105804Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285113994Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285133605Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285146145Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285155425Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285159905Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285165195Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.285169425Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.441909214Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:25.441941475Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"05effe78-0c08-40b4-916b-cde344744ec3 (zpool)"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:25.702937846Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.644472071Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.655479497Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.662955963Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.663004314Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.663015254Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.663040125Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.663052065Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.663169147Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.663178817Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.695080692Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_05effe78-0c08-40b4-916b-cde344744ec3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.701814244Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:26.705356853Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Looking for unencrypted datasets in oxp_05effe78-0c08-40b4-916b-cde344744ec3","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.861568612Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.868839134Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.868880455Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.868923096Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.868932636Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.868962627Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.878805049Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.8788353Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"866d818f-82bb-444c-992a-1351dec91efa (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.969472804Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977201915Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977253246Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977261987Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977334818Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977364409Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977481641Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:26.977490251Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.015118368Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_866d818f-82bb-444c-992a-1351dec91efa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.024116494Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_866d818f-82bb-444c-992a-1351dec91efa","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.201831424Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.209450853Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.209488854Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.209494764Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.209500084Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.209506844Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.218610762Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.218636533Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.310107644Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.317738143Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.317770604Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.317778874Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.317799904Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.317825615Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.317987208Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.318000128Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.352544675Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.35994842Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.532014089Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.540036836Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.540065447Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.540073767Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.540078307Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.540091927Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.549606134Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.549623414Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.647831457Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655474596Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655520067Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655527288Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655554498Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655561068Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655694081Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.655699231Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.691248807Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_6eaf0007-0aa9-44b6-8bac-b01569f4c6a3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.699328445Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:27.709453384Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Looking for unencrypted datasets in oxp_6eaf0007-0aa9-44b6-8bac-b01569f4c6a3","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.872870513Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.880739987Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.880791758Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.880799428Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.880804699Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.880809009Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.891000958Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.891030919Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.987005658Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.994710069Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.9947525Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.99475933Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.994795441Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.994806521Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.995001855Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:27.995024875Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.032864406Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_775439b8-d6ea-4b72-9708-a30c4c7c6d6f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.041049476Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_775439b8-d6ea-4b72-9708-a30c4c7c6d6f","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.211731379Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.219570152Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.219613873Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.219624923Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.219634773Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.219640373Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.229048818Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.229077018Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"e4f77dae-534d-4b73-b38a-890d525d3747 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.325286932Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.332782579Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.33282529Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.33283345Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.33284425Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.33285689Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.332967672Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.332978402Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.367675152Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_e4f77dae-534d-4b73-b38a-890d525d3747/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.374927694Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_e4f77dae-534d-4b73-b38a-890d525d3747","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.549059264Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.557095331Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.557137352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.557145842Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.557153932Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.557159012Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.566073237Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.566092517Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.663080266Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.671732466Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.671760936Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.671766926Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.671786137Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.671795987Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.67196834Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.67197678Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.708135888Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:28.711985554Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Automatically destroying dataset: oxp_1b55399c-9dbb-4d32-bd51-426a7996ea24/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.715836339Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_1b55399c-9dbb-4d32-bd51-426a7996ea24","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.887622053Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.895085839Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.89512495Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.89513352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.8951407Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.89514725Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.905282329Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:28.905311999Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.001664815Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010476358Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010518829Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010525009Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010539329Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010548839Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010716443Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.010724343Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.0498914Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_aad99d77-5c0c-4ae7-8cc1-ced403d6b98a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.05806422Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_aad99d77-5c0c-4ae7-8cc1-ced403d6b98a","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.233863682Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.241864739Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.24192918Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.2419383Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.2419458Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.24195224Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.251917655Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.251966756Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"f4912882-4c2a-4c0d-b986-5cbff960587e (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.34663262Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.354226869Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.35426706Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.35427379Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.35431768Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.354328491Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.354516735Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.354530045Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.390912987Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_f4912882-4c2a-4c0d-b986-5cbff960587e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.39873007Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"}
{"msg":"Looking for unencrypted datasets in oxp_f4912882-4c2a-4c0d-b986-5cbff960587e","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.569251039Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.576875718Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.57693153Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577029672Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577158364Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a85a3ef-48f8-4333-ae7e-46dfaa7fa523 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6dd67a4e-189f-47cd-a6e2-0eff97293dab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577214485Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577221535Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577312367Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577520211Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577534291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577543462Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:886","generation":"Generation(1)"}
{"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577548802Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:891"}
{"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577554712Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:896"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577583942Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577594143Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:902"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577633623Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:803","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577640164Z","hostname":"sock","pid":1562,"sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:907"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.577717655Z","hostname":"sock","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"6eb7af0e-c8aa-48e8-95bb-677cb8f4cbd5","remote_addr":"[fd00:1122:3344:101::1]:63462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4292820,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.60964868Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.609767162Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:62268"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.610038318Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.610043708Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.657860934Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.657905815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.705837944Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.705876215Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:29.71737508Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.754578248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.754620619Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.754627299Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.75466466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.799542449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.799586449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.79959239Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.851926814Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.855557575Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.859136625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.872366655Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.876369943Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:29.879941743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:30.719852239Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Profile for oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService3'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:1::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:1::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:1::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:1::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:31.396967547Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:31.397034178Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"}
{"msg":"Profile for oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService1'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:2::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:2::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:2::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:2::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:31.875965266Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:32.351307763Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"}
{"msg":"Profile for oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService2'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:3::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:3::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:3::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:3::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:32.351449316Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:32.351458997Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:32.355343262Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"}
{"msg":"Booting oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:32.355380513Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:32.359625476Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 51.30061ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.477232359Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 35.863528ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.483687355Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 37.636147ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.489130132Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 132.196478ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.525580896Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 145.528795ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.533776366Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 65.68834ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.547006655Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 253.255053ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.735858193Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 164.690817ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.759064848Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 147.073015ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.919850566Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 592.424034ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:32.946826954Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 458.321473ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:33.002163637Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 521.576482ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:33.099801409Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 571.473072ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:33.474909564Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 702.543989ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:33.55129187Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 410.42864ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:33.631120553Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 756.030551ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:34.053774769Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 1.162512513s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:34.062685643Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 1.405567001s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:34.263852162Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 1.15309281s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:34.821648464Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 1.251217681s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.233347245Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.298868708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4211"}
{"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.298910749Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221","gateway":"fd00:1122:3344:101::1"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.304687052Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4300"}
{"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.434167847Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4348"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.434201128Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4375"}
{"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.475211031Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4388"}
{"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.475248982Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4252"}
{"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.527466454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4294"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.527507185Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4355","rack_subnet":"fd00:1122:3344::"}
{"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.54770048Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4366"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.547753211Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4408"}
{"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.637133821Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4440"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.637172582Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 1.199170507s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.678110934Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.748944351Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4487"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:35.748994562Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.781560889Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"Failed to get switch ID from MGS (retrying in 29.63559ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.781695632Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Failed to get switch ID from MGS (retrying in 52.567226ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.812183679Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Failed to get switch ID from MGS (retrying in 126.206105ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.86639865Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 598.019722ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.982674847Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Failed to get switch ID from MGS (retrying in 330.493544ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:35.99300834Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.325130743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.33567991Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:54464"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.336052477Z","hostname":"sock","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"3f85cc38-20af-4573-959e-5b0614214a63","remote_addr":"[fd00:1122:3344:101::2]:54464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87,"response_code":"200"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.358143369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.358266672Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.358643249Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"0"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.358660829Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.396352407Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.444892058Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.44497035Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.44498461Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.44499278Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.44502582Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445033941Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445043491Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445051351Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445060851Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445068451Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445077901Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445085782Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445091602Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445097202Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445102972Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445108572Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445118602Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445124333Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445130073Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445135713Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445141293Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445146983Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:36.445152363Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"wait for service Some(\"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838\") failed: Property not found. retry in 1.474559347s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:36.492287776Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 647.402593ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:36.588614692Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87\") failed: Property not found. retry in 1.257920072s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:36.887557766Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb\") failed: Property not found. retry in 1.431049397s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:37.245004494Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.696572346Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_ccbc07cd-fbf0-43da-b57b-17b3debd7838"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.696637417Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_c7342971-675e-4b24-995a-35efbafcac87"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.696644687Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_internal_dns_285489bf-dd9f-4685-a3f1-46733c794cfb"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.696660708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.696926263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.697214309Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"99f5e8a2-10f8-47ee-b88a-59c7901d0cb0","remote_addr":"[fd00:1122:3344:101::1]:62268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9087386,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.699204437Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.748399381Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.796408251Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.844139906Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:591"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.844149346Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.877263504Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.878135181Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.878145871Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.907606138Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.90770312Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65086"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.907894064Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.907900734Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.929237292Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.929289303Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.946472499Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.94650142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.963998753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.964043393Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.964226497Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.96436076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:38.975195402Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.008152757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.008884191Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.008894892Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.039605703Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 223, 79]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.039616923Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"boundary_ntp"}
{"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.039935009Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.03994813Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.040283726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.040674024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.064871238Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.081770428Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService4'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value='fd00:1122:3344:100::/56'/>\n        <propval type=\"boolean\" name=\"boundary\" value='true'/>\n        <propval type=\"astring\" name=\"boundary_pool\" value='boundary-ntp.control-plane.oxide.internal'/>\n        <propval type=\"astring\" name=\"server\" value='0.pool.ntp.org'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte0'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.3.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.3.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.701509003Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:39.701630216Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 41.884875ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:39.834796664Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 93.495881ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:39.884288433Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 146.390482ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:40.00371386Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 417.468809ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:40.162211364Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 475.517419ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:40.588815157Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 1.128076891s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:41.073516737Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 1.124862757s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:42.21106056Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 925.492782ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:43.345756568Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25\") failed: Property not found. retry in 641.688766ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:44.280174424Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.941970092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.942043874Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.942464662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.942817379Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cbb35c06-0afd-4bef-8a71-ed9294653692","remote_addr":"[fd00:1122:3344:101::1]:65086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6035064,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.945536042Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.976859255Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.977020579Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65029"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:44.985383282Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:44.985509505Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1e8f2fcb-2070-4802-9805-e489ee5ef05f","remote_addr":"[fd00:1122:3344:101::1]:65029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8307,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:44.98681572Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:45.073828174Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:45.073943156Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59692"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:47.081975634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.082093456Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ec8942c4-ec59-49a5-b5e0-8aec4f74d84f","remote_addr":"[fd00:1122:3344:101::1]:59692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":2008023,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:47.085737718Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.26410921Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.264240733Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45972"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:47.272863762Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.273016755Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ecea9fe4-5884-4061-9da2-8f23b4936678","remote_addr":"[fd00:1122:3344:101::1]:45972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8640,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:47.276050404Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.516634035Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.516731217Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37823"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:47.524373707Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.524453388Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"05934969-4c70-4395-b95b-569140a4a9e4","remote_addr":"[fd00:1122:3344:101::1]:37823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7587,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:47.525986888Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.789307804Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:47.789410526Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37245"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:48.798510034Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:48.798749939Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"27d7e1a0-782c-4ba1-a07a-1b6c103a4f8e","remote_addr":"[fd00:1122:3344:101::1]:37245","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":1009178,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:48.801670496Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.02203548Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.022150873Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33191"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:49.030623498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.030741961Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"4f898ee9-e2b6-4181-a801-4a42cc03fb80","remote_addr":"[fd00:1122:3344:101::1]:33191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8503,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:49.032653828Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.296879242Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.296981914Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57885"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:49.305110703Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.305277976Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"c1866df3-b955-4c26-8f63-026f5a143b81","remote_addr":"[fd00:1122:3344:101::1]:57885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8219,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:49.307365967Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.547636502Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:49.547728314Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54855"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:50.556616539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:50.556802722Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"a01e0f37-f257-4d89-8502-ec17a9136527","remote_addr":"[fd00:1122:3344:101::1]:54855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":1009005,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:50.559471615Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:50.941717969Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:50.941823241Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55155"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_43a6fa50-4493-4748-921d-7ff7c23c0e25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:14:50.94993047Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:50.950043232Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"6293a0e2-4e78-466d-b7be-54851340a233","remote_addr":"[fd00:1122:3344:101::1]:55155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8097,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:50.951696204Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:51.371435273Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:51.371541795Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49790"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:51.380564641Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"aac1c1dc-5d71-4d8e-8b78-d209d485fa8e","remote_addr":"[fd00:1122:3344:101::1]:49790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8909,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:51.382892367Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:51.382897567Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.030392036Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.030482658Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50853"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.038495354Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"a62b2d77-36ee-47ba-a16b-fd1d217abc93","remote_addr":"[fd00:1122:3344:101::1]:50853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7913,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.041264809Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:52.041271049Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.439340193Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.439435755Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59328"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.447385101Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1092fd6c-177b-4c73-9244-666e4205f821","remote_addr":"[fd00:1122:3344:101::1]:59328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7854,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:52.450304268Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:52.450311288Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.050609201Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.050728634Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:43296"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.071568482Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"c73c190d-ef62-4c50-aeeb-3e924a3031d1","remote_addr":"[fd00:1122:3344:101::1]:43296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":20706,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2616771612, ip_addr: 155.248.196.28, stratum: 3, ref_time: 1724944492.8194437, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.074440598Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.074446418Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1266"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.106508496Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.106638618Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41420"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.106988225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.106997245Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.127697731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.127741002Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.146877416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.146910677Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.165328267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.165366878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.220022609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.220058609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.272423324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.272459165Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.324169468Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.324202778Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.376728497Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.376761358Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.426075783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.426122764Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.426342489Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.426504492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.458448877Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.458492938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.458504998Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.458511708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.458519348Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.458654051Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"85c8d87d-d078-4e2b-91ea-ae7392273da6","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized)]","latency_us":351910,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.458852165Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"145.186357ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85c8d87d-d078-4e2b-91ea-ae7392273da6\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85c8d87d-d078-4e2b-91ea-ae7392273da6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.605041727Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.606312812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.606351793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.624871925Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.624898356Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.643152003Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.643186904Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.66034688Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.660385031Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.678605197Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.678642718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.697856234Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.697887765Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.715833686Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.715864147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.73289537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.732923891Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.752884452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.752924263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.753106886Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.753241289Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.784950319Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.785015671Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.785026051Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.785036221Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.785045892Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.785155164Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9d447e89-6706-49dc-8d74-5cfaba5934d7","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized)]","latency_us":179941,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:53.785310317Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"85.08457ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d447e89-6706-49dc-8d74-5cfaba5934d7\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d447e89-6706-49dc-8d74-5cfaba5934d7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.871676128Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.871898242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.871907452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.892063877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.892110078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.911188011Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.911244552Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.928787116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.928831797Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.946744038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.946786348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.964187879Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.96422525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.982516228Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:53.982556179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.000398338Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.000437539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.018240858Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.018299739Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.018531343Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.018731077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.050629702Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.050670313Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.050679393Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.050685973Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.050707943Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.050813705Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c1c2ba53-6206-4a53-a4d9-a54162899729","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":179023,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.050952448Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"141.594144ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1c2ba53-6206-4a53-a4d9-a54162899729\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c1c2ba53-6206-4a53-a4d9-a54162899729\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.066132845Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.067349519Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.067362219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.086247789Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.08628726Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.103923555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.103961636Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.122328455Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.122359186Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.140978751Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.141010731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.159116536Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.159141166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.177541837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.177578297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.196013378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.196046099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.215261585Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.215308146Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.215554481Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.215700084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.248099808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.248138629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.248150819Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.24816071Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.24817079Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.248257541Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"f862700a-56ee-4b76-8fb8-dc79491b87e0","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized)]","latency_us":181957,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.248394974Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"190.262652ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f862700a-56ee-4b76-8fb8-dc79491b87e0\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f862700a-56ee-4b76-8fb8-dc79491b87e0\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.438887443Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.439171539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.439184929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.45761627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.45765066Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.475743135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.475782566Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.493615075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.493657096Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.5112593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.511299041Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.529095939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.52913208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.547474409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.54750875Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.565582094Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.565644575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.584906242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.584948683Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.585144737Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.58527766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.616695845Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.616736466Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.616746906Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.616756276Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.616766776Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.616867238Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"f974c56c-db90-4cc4-b1d7-f15c6abfbc37","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized)]","latency_us":177855,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.617007821Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"147.416271ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f974c56c-db90-4cc4-b1d7-f15c6abfbc37\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f974c56c-db90-4cc4-b1d7-f15c6abfbc37\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.765395077Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.765753173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.765767834Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.783573332Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.783624774Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.801500344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.801537154Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.819788991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.819828382Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.83758526Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.837627501Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.856268456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.856308847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.87385792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.873892121Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.892321902Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.892372103Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.909486428Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.909537949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.909769044Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.909889016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.943260339Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.94330184Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.94331092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.94331767Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.94332617Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:54.943436023Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"de5a95ab-1c97-4a63-92c2-9511b6324c1e","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":177890,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:54.943608196Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"332.283817ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"de5a95ab-1c97-4a63-92c2-9511b6324c1e\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"de5a95ab-1c97-4a63-92c2-9511b6324c1e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.277904079Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.27824902Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.278266374Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.296554952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.296596577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.314846876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.31488341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.331729631Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.33176446Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.351324037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.35137484Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.370359959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.370400948Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.387840666Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.387875345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.40416369Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"128.036551ms"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.406451195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.406461564Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.424224914Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.424277058Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.424483462Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.424660297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.456160027Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.45626813Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.456276999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.45628539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.456293743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.456375635Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"04ad6c4f-9818-4235-82f9-0cce29b1bf2b","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":177825,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:55.456510019Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"434.328132ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04ad6c4f-9818-4235-82f9-0cce29b1bf2b\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04ad6c4f-9818-4235-82f9-0cce29b1bf2b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.893064505Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.893439572Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.89345852Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.914299952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.914334913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.933171193Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.933207185Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.952201815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.952237094Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.971430301Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.971462007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.989529277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:55.989558631Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.00931585Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.009350998Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.027330632Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.027368142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.045956195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.046006052Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.046217969Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.046369041Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.078816718Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.078928749Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.078938649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.078949356Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.078958928Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.079038195Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"98bb2684-7f4a-4f8e-982f-8f2fbad817d3","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":185327,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.079168154Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"232.167435ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98bb2684-7f4a-4f8e-982f-8f2fbad817d3\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"98bb2684-7f4a-4f8e-982f-8f2fbad817d3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.313202275Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.313645324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.313665762Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.333018937Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.333060619Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.351669577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.351710406Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.370172159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.370208477Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.388349774Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.388385803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.405942673Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.405973386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.423161151Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.423206095Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.441415643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.441456238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.459061476Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.459113649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.459312665Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.459462069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.49249344Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.492545406Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.492555128Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.492564344Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.492580647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:56.492685134Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cf49dc43-9af9-4048-9c0c-335befd6f628","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":178814,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:56.492836675Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"581.039957ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cf49dc43-9af9-4048-9c0c-335befd6f628\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cf49dc43-9af9-4048-9c0c-335befd6f628\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.076496502Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.076769037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.076778102Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.095509457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.095536532Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.113511238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.11354797Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.131516316Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.131558073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.151023613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.15105578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.170141385Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.170173794Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.189749753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.189783635Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.207466784Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.207510012Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.226392953Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.226441095Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.226609211Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.22672311Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.258949172Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.259009501Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.259020058Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.259029883Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.25910785Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.259180911Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3ee439aa-ce1c-40af-8f47-ecae218d7635","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":182042,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.259312679Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"325.931023ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ee439aa-ce1c-40af-8f47-ecae218d7635\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ee439aa-ce1c-40af-8f47-ecae218d7635\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.58762114Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.588055654Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.588073195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.607471858Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.60752315Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.626905606Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.626945994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.644852305Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.644893716Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.663532166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.663575274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.682925896Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.682963274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.702838259Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.702882397Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.721425567Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.72146889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.740470274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.740524491Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.740683344Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.740799887Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.7736278Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.773677385Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.773688739Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.773698068Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.773707264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:57.773805526Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"51efeef9-b20e-44b3-b928-022c171f1a64","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized)]","latency_us":185487,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:57.773954169Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"810.891211ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51efeef9-b20e-44b3-b928-022c171f1a64\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"51efeef9-b20e-44b3-b928-022c171f1a64\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.587427318Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.587701647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.587710393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.607742556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.607773729Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.627709064Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.627814136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.647711847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.64774211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.666244289Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.666270755Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.684015577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.684042662Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.701770722Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.701800563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.721197577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.721229902Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.73913701Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.739177117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.739368727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.739491804Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:58.771311581Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:58.771351801Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:58.771361072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:58.771368385Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:58.771377733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:58.77148145Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b02513c6-22fa-46e3-872d-a0a1d49f4498","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":183421,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:58.771647906Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"791.599088ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b02513c6-22fa-46e3-872d-a0a1d49f4498\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b02513c6-22fa-46e3-872d-a0a1d49f4498\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.566339956Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.566733874Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.566745715Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.58617409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.586205263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.605360051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.605393163Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.623770229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.623802629Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.642697475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.642722356Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.661096092Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.661122136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.68016555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.680196721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.698046221Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.698072649Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.715560464Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.715600027Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.715859774Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.715964701Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:59.748488032Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:59.748525608Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:59.748532405Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:59.748541667Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:59.748550124Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:14:59.748647588Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2af92ffd-a89f-4fe7-92bc-6bcef4a052a3","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized)]","latency_us":181609,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:14:59.748795154Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.174093048s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2af92ffd-a89f-4fe7-92bc-6bcef4a052a3\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:14:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2af92ffd-a89f-4fe7-92bc-6bcef4a052a3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.92784021Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.928713467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.928722111Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.953149749Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.953191684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.976186772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.976227601Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.998364471Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:00.998463551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.020631508Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.020663365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.043169242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.043229421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.065117448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.065161165Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.086822829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.086867473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.109651877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.109771983Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.109936864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.110050228Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:01.144064338Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:01.144109911Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:01.144116745Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:01.144124076Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:01.144150964Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:01.144225572Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"699a7b0e-4d07-432b-8f95-86d7831b8a8f","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":215575,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:01.14435764Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.078472214s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"699a7b0e-4d07-432b-8f95-86d7831b8a8f\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"699a7b0e-4d07-432b-8f95-86d7831b8a8f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.227233103Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.227590748Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.227607276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.246660421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.246705356Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.265583542Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.265621352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.283051163Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.283094138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.301904739Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.302012397Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.321614886Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.321663084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.340380323Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.340421837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.359043055Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.359150832Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.378702675Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.378751426Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.378920629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.379035729Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:02.411810633Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:02.411849165Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:02.411858222Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:02.411864719Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:02.411873194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:02.411989249Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0303b6c4-de46-4efc-8435-52f5495344f5","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized)]","latency_us":184132,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:02.412147681Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.305149646s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0303b6c4-de46-4efc-8435-52f5495344f5\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0303b6c4-de46-4efc-8435-52f5495344f5\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.722456352Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.722897531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.722936195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.743788068Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.743893193Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.763776387Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.763815735Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.782784392Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.7828197Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.801631941Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.801669245Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.819816188Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.819854053Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.838752156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.838792348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.857108066Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.857210207Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.875165668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.87526849Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.875457362Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.875569584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:03.908619532Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:03.908679974Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:03.908690521Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:03.908697478Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:03.908706524Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:03.908811714Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2698c763-101e-41c6-bfa7-f0039a78dc57","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":185641,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:03.908970398Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.299137698s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2698c763-101e-41c6-bfa7-f0039a78dc57\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2698c763-101e-41c6-bfa7-f0039a78dc57\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.002544794Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50006"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.002662968Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"db8f9734-c022-4d98-bb90-d9034fd7d4ed","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50006","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":53,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.213169941Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.213407424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.213418263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.235731122Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.235782312Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.255570626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.255628452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.274010418Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.274064063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.292519856Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.292561435Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.311331257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.311373445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.330310453Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.330352463Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.348245629Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.348289402Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.367154183Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.367276848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.367465954Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.367587073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:05.400510043Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:05.4005578Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:05.400570055Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:05.40058107Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:05.400593445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:05.400658566Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"afcba35c-51f3-48a8-b2a0-6a1760ef2a8b","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized)]","latency_us":186860,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:05.400787061Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.555828637s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afcba35c-51f3-48a8-b2a0-6a1760ef2a8b\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"afcba35c-51f3-48a8-b2a0-6a1760ef2a8b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:07.964140107Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:07.964555079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:07.96456798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:07.985131425Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:07.985162344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.004090466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.004121001Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.024490897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.024569348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.044078227Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.044107683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.064872628Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.064896525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.08299158Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.083018516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.102307006Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.102342398Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.120158534Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.120197675Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.120391488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.120534535Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:08.151513889Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:08.151543694Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:08.151552272Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:08.151558694Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:08.151564909Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:08.15168793Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e957539f-e95f-4d49-a9b1-27db29bbb93a","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":186909,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:08.151927344Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.781391694s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e957539f-e95f-4d49-a9b1-27db29bbb93a\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e957539f-e95f-4d49-a9b1-27db29bbb93a\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:10.609198766Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"748.301471ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.941351392Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.941691861Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.941702578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.962050288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.962090039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.980420514Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.980459355Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.997054684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:10.997088894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.016912594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.016949681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.033550631Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.033586126Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.052075801Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.05211363Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.069348911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.069381677Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.085865912Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.085914522Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.086093916Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.086206747Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:11.117299693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:11.117344609Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:11.117352184Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:11.117360622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:11.117367381Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:11.117476799Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c552dd24-a94d-4ea4-b71c-162092a39fb3","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":175548,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:11.117663009Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.671100349s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c552dd24-a94d-4ea4-b71c-162092a39fb3\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c552dd24-a94d-4ea4-b71c-162092a39fb3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.797438753Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.797757284Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.7977686Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.819734435Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.819763479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.840364535Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.840400058Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.859028825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.859056491Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.878295122Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.87833702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.896655748Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.896686395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.915456367Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.91548935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.933491831Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.933522432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.95201805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.952059928Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.952283676Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.952504457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:13.984421594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:13.984538191Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:13.984547369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:13.984556257Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:13.984566682Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:13.984650271Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2c394505-4505-4f58-925c-4f835ab2e97f","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized)]","latency_us":186548,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:13.984779817Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"3.786444778s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c394505-4505-4f58-925c-4f835ab2e97f\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c394505-4505-4f58-925c-4f835ab2e97f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.782588194Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.783082293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.783094472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.804871401Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.804919168Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.825073354Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.825109597Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.844011185Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.84404593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.862266216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.862299377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.88192926Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.882037843Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.900784718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.900821131Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.919240872Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.919269701Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.937688277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.937724896Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.937896922Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.938031202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:17.9715358Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:17.971585648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:17.971593308Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:17.971604117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:17.971620712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:17.971757935Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"260f9be6-ae36-49cb-857c-db4d767e5b97","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":188447,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:17.971896819Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.006830684s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"260f9be6-ae36-49cb-857c-db4d767e5b97\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"260f9be6-ae36-49cb-857c-db4d767e5b97\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:21.990359812Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:21.990689125Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:21.99070226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.011446032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.011486692Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.030424539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.030478822Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.049482846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.049523394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.068744725Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.068781992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.086291498Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.086389231Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.104690511Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.104738371Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.122259992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.122289571Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.141190453Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.141220763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.141413151Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.141628297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:22.174009315Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:22.174052019Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:22.174059773Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:22.17406701Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:22.174074416Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:22.174203907Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9a564963-bd75-49af-bce0-8baad62bf747","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized)]","latency_us":183227,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:22.174356263Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.723906088s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a564963-bd75-49af-bce0-8baad62bf747\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9a564963-bd75-49af-bce0-8baad62bf747\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:26.437952436Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.313760181s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.913204736Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.913671768Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.913688353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.934693173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.934734198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.955239855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.955277056Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.973400223Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.973431209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.993785302Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:26.993816241Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.012897028Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.012930487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.030652144Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.030748539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.049152577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.049190613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.067000285Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.067044246Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.067279591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.067429735Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:27.099673197Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:27.099709048Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:27.099715602Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:27.099723186Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:27.09973078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:27.099824494Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"14a1ec27-5e70-4beb-8308-f762a962b641","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":185924,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:27.100024292Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"3.181316361s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14a1ec27-5e70-4beb-8308-f762a962b641\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"14a1ec27-5e70-4beb-8308-f762a962b641\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.290933871Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.291229293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.291240243Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.312620793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.312661547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.331563151Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.331662043Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.349423956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.349459563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.368659977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.368757457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.387186297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.387232198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.404702957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.404733895Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.424203279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.424254543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.442147828Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.442215452Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.44243844Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.442594631Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:30.474581519Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:30.474611191Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:30.474618241Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:30.474625019Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:30.474630382Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:30.474737268Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"79408470-96f8-42ab-a15f-48f91fbb0cb2","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":183197,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:30.474856352Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.032582996s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"79408470-96f8-42ab-a15f-48f91fbb0cb2\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"79408470-96f8-42ab-a15f-48f91fbb0cb2\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.519040701Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.519336497Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.519346951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.540218627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.540261763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.55896575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.559076208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.576877142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.576912571Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.595069547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.59517461Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.614322809Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.614358979Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.631679467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.631720024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.649638613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.649690636Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.668424118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.668476151Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.668633663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.66874713Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:34.70073956Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:34.700771652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:34.700778186Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:34.700784861Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:34.700793505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:34.70092247Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ac16d1da-b833-4958-bc98-63cc320dbb62","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized)]","latency_us":181281,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:34.70109352Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"10.497327632s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ac16d1da-b833-4958-bc98-63cc320dbb62\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ac16d1da-b833-4958-bc98-63cc320dbb62\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:35.118439429Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50021"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:35.118588799Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a857a108-0c5a-45b1-b513-d9dd99a18f47","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50021","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:42.832902669Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.468100608s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.228279048Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.228578107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.228588964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.250803079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.250851643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.270434627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.270472447Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.2889491Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.289050492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.30683911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.306871604Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.326228283Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.326270425Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.345500507Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.345540999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.364331546Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.36436465Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.381858087Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.381910588Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.382211185Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.382427157Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:45.415034815Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:45.415064356Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:45.415069269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:45.415074256Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:45.415080209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:45.415197175Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"01432e4b-f743-4c51-82a5-e9a183490fc6","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized)]","latency_us":186306,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:45.415426494Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"12.613846732s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"01432e4b-f743-4c51-82a5-e9a183490fc6\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"01432e4b-f743-4c51-82a5-e9a183490fc6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.064932479Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.06580145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.065807901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.086398107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.086429018Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.106402625Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.106431921Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.125636567Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.125736971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.142308824Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.14233019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.160924225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.160949238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.178314475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.178342581Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.195405644Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.195427254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.213358601Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.213400349Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.213613466Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.213719246Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:58.24339052Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:58.243431846Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:58.243441878Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:58.243449547Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:58.243458313Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:15:58.243554793Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"04dd8666-f5da-4a79-8972-780556451c22","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized)]","latency_us":178026,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:15:58.243711152Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"15.299847273s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04dd8666-f5da-4a79-8972-780556451c22\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:15:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04dd8666-f5da-4a79-8972-780556451c22\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:00.402326889Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"4.607903928s"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:05.235302273Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47214"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:05.235417326Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"98a2376a-aaef-414a-bdb2-0c37482884d8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47214","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.585783599Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.586080146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.586093168Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.607553093Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.607591091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.626246313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.62628853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.644683296Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.644726835Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.663408708Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.663452359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.68176858Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.681804403Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.699988115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.700021998Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.717888226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.717921808Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.73646867Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.736512603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.736776186Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.736985518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:13.770574395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:13.770613068Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:13.770619809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:13.770628293Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:13.770634528Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:13.770746674Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"dd2a1f2f-f49e-4223-80cd-b2b4b73496ff","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":184378,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:13.770982286Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"7.230124239s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd2a1f2f-f49e-4223-80cd-b2b4b73496ff\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:16:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dd2a1f2f-f49e-4223-80cd-b2b4b73496ff\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:20.100600276Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"10.243453253s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.022290234Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.0225719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.022580675Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.044159576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.04419943Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.063488089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.063523593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.082144376Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.082188869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.100167459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.1002021Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.118543321Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.118586174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.13691426Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.136949117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.154334464Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.154364606Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.172496712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.172549485Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.172770815Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.172976884Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:21.204691518Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:21.204733097Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:21.204742678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:21.204750413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:21.204760228Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:21.204881075Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"fd55732d-1523-431d-a6ef-8a81c8c427a2","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":181996,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:21.20511687Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"13.356932672s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fd55732d-1523-431d-a6ef-8a81c8c427a2\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:16:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fd55732d-1523-431d-a6ef-8a81c8c427a2\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.59911948Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.599489149Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.599501993Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.621653647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.621678219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.640808931Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.64083265Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.65845786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.658488844Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.676575076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.676601982Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.695217767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.695240445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.71321063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.71324065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.731241091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.731280954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.749150248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.749184091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.749408777Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.749518223Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:34.781638599Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:34.781748647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:34.781758144Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:34.781767369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:34.781776304Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:34.78185574Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b1227758-043f-43e5-be5a-46dd999b5cd7","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized)]","latency_us":182066,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:34.78199994Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"17.750605211s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1227758-043f-43e5-be5a-46dd999b5cd7\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:16:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1227758-043f-43e5-be5a-46dd999b5cd7\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:35.354562094Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49216"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:35.354683521Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8656a3b5-4c05-4166-870a-8f7aa7c7f2b5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49216","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:45.450408737Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"21.684470562s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.58243131Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.58271835Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.582794517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.604923168Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.604968796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.623633455Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.623666568Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.640846319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.640957336Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.658408408Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.658445693Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.677463812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.677495199Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.696287099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.696319977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.714002681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.714042506Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.731886386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.731950456Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.732246375Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.732359139Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:52.764275295Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:52.764321825Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:52.764331182Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:52.764339732Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:52.764346688Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3254","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:16:52.764416702Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4862bad0-989d-4b77-acfa-d1709eb43602","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\", TimeNotSynchronized), (\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\", TimeNotSynchronized), (\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\", TimeNotSynchronized), (\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\", TimeNotSynchronized), (\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\", TimeNotSynchronized)]","latency_us":181341,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:16:52.764993678Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"32.567222173s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4862bad0-989d-4b77-acfa-d1709eb43602\", \"content-length\": \"124\", \"date\": \"Thu, 29 Aug 2024 15:16:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4862bad0-989d-4b77-acfa-d1709eb43602\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:05.469215086Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42234"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:05.46928389Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"24b29b66-5472-4159-ba7d-ed23287d0f46","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42234","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:22.269784455Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"24.73035272s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.421364316Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.421624248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.42163651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.444260535Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.444299218Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.462983356Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.463010132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.482647233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.482681143Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.502990074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.503018013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.521845986Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.521875181Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.541127684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.54115911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.561665922Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.561704717Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.579263994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.579294097Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.579513008Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.579614992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.661691812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.661742729Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.661754785Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.661762369Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.661768848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.752121833Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.756541963Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.760688156Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.764936877Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.769293012Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.773639218Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.778482618Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.782900976Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.787597734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:25.792068898Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService8'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::7'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::7]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='5dc3d2e8-7d3d-4338-9928-132828621316'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::7]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::7]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.652225711Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService5'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::5]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='0dac43bc-7b48-477d-9f9f-6fd3fe599f5e'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::5]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::5]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.652405784Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.652422435Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.658376461Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService6'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::4'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::4]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='70a742d6-87c1-4c50-8096-5f2e5549f875'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::4]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::4]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.716578911Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.716722864Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService9'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::3'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::3]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='6d2f290a-abdf-4ede-9ef2-830ec6ce9b33'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::3]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::3]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.725480095Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.725634418Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService7'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::6]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='6a3e1632-3f67-4c67-a09b-2ee7c055f6de'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::6]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::6]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.745752342Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:28.745879265Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 43.129385ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:28.810723075Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 26.573782ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:28.819092308Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 43.925953ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:28.853537322Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 50.507784ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:28.860916566Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 72.130124ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:28.868157528Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 62.297876ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:28.963544266Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 194.634023ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:29.716383117Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 79.586642ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:29.903746616Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 74.841388ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.42537174Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 217.10954ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.67457066Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 77.9811ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.752248411Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 424.351178ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.765524911Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 184.546234ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.779201399Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 91.350338ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.791865987Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 264.453475ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.842730062Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 165.167171ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.896603937Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 286.189968ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.907336717Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 280.247183ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:31.977005721Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 276.995414ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.074847787Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 573.428016ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.117444431Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 753.607736ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.20165076Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 1.169873683s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.211922751Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 475.498528ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.267894797Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 483.869852ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.363624471Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 901.582681ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.700777463Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 555.949166ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.75217384Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 1.240512775s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.856408421Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 760.317013ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:32.964810883Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.430098043s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:33.316612182Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 867.496446ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:33.390424227Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 543.840089ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:33.612646639Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 862.718664ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:33.733985055Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 1.472606006s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:34.107499408Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 590.807355ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:34.167012413Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 883.259699ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:34.275493047Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 1.305317161s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:34.621424661Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.01262188s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:34.758691639Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 1.384612632s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:34.768927409Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 1.302232228s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:35.170995902Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:35.556686574Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53764"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:35.556817447Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"731b0e93-c93c-4013-a851-c1599d30a4d8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53764","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 511.401939ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:35.588388785Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 949.724495ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:35.779417466Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 1.432813346s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:35.936238666Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 704.205216ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:36.108596881Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 726.699692ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:36.161122289Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 600.724781ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:36.481255018Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 527.325812ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:36.739674498Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 1.067150647s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:36.82456096Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 568.854797ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:36.897406996Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316\") failed: Property not found. retry in 1.319081848s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:37.09215473Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.361893154s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:37.276111152Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e\") failed: Property not found. retry in 757.506357ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:37.379357924Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33\") failed: Property not found. retry in 1.412273509s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:37.476305152Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 1.099652738s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:37.902516348Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 937.213888ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:38.650152138Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 927.275596ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:39.028506577Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.255926741s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:39.598158762Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 694.244503ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:39.966245219Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 880.992795ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:41.141110365Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 1.1871751s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:41.158117648Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 507.172547ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:42.031742705Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 983.160715ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:42.353962985Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 955.337369ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:42.549556455Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 551.332329ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:43.3459856Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 845.328482ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:43.513873738Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 681.396349ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:43.908333511Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.24736394s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:44.369318678Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 665.676033ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:44.5981876Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 937.791085ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:45.273552655Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.422654582s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:45.625582018Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 1.116622914s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:46.220413976Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875\") failed: Property not found. retry in 1.370573841s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:47.058970158Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 577.721189ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:47.347264243Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de\") failed: Property not found. retry in 531.135141ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:17:47.935637984Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.485806718Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_0dac43bc-7b48-477d-9f9f-6fd3fe599f5e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.48588117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_70a742d6-87c1-4c50-8096-5f2e5549f875"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.4858912Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_6a3e1632-3f67-4c67-a09b-2ee7c055f6de"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.4858981Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_5dc3d2e8-7d3d-4338-9928-132828621316"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.485920301Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_cockroachdb_6d2f290a-abdf-4ede-9ef2-830ec6ce9b33"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.485930861Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.486819848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.487704236Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"393d84ba-3987-42a1-9baa-ad6590896c95","remote_addr":"[fd00:1122:3344:101::1]:41420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":23065599,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.520341655Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65093"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:48.549633668Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3686"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:17:49.393005533Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3704"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.437598568Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3727"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.43773262Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"17f05b7b-48d8-4b00-bdab-314d95df55bc","remote_addr":"[fd00:1122:3344:101::1]:65093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":23917746,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.43822818Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44615"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.438559976Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a7fffcd1-51dc-4bc7-a1b1-955a2df5477f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44615","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:12.443147206Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"89.726462634s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 285489bf-dd9f-4685-a3f1-46733c794cfb, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c7342971-675e-4b24-995a-35efbafcac87, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b8c196d9-a607-40a2-9036-1e2d2f60dc07, kind: Service { id: 43a6fa50-4493-4748-921d-7ff7c23c0e25 }, name: Name(\"ntp-43a6fa50-4493-4748-921d-7ff7c23c0e25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 223, 79])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70a742d6-87c1-4c50-8096-5f2e5549f875, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dc3d2e8-7d3d-4338-9928-132828621316, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }, OmicronZoneConfig { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a752201f-8e19-45ac-9901-65317e43fc2e, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 776ef919-ac0a-4b14-9dfe-deac75d16696, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 709ff80f-750f-45c6-b668-97969f356dc0, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4c2f62ac-cb40-467c-ba9f-de7aa05f3691, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 45d4b7eb-c738-4794-9125-01d0ae07feb2, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: bbf9c6ad-fa23-470f-b608-67105cf521a9, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b880f4d-eaa0-4b77-919b-817d6a980247, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa3cf0ac-1c68-4e1c-a127-490862a19932, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a83427f-50d0-47ad-8765-fad7f4c19873, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External } } } }, OmicronZoneConfig { id: f389808b-a883-46ea-a396-af6431b5b6e3, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 23eb92fe-33fd-4e95-a819-b3349525e67a, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External } } } }, OmicronZoneConfig { id: d98c3f2a-4c9c-4655-9407-3839f53cf851, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External } } } }, OmicronZoneConfig { id: 57f24d92-52ba-4db7-a1b1-c2a25e015a03, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9becf607-a424-4af6-bd2e-178c196b8348, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool), kind: External } } } }, OmicronZoneConfig { id: 3da8a36e-1727-4d21-9b85-8be498e95b8c, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.473804867Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.47395724Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65026"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.474351617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.474358228Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 285489bf-dd9f-4685-a3f1-46733c794cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.498599162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.498648963Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: ccbc07cd-fbf0-43da-b57b-17b3debd7838, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.520715135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.520769136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c7342971-675e-4b24-995a-35efbafcac87, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.54239259Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.54243803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6d2f290a-abdf-4ede-9ef2-830ec6ce9b33, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.56233188Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.562370521Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 70a742d6-87c1-4c50-8096-5f2e5549f875, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.581244961Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.581285691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 0dac43bc-7b48-477d-9f9f-6fd3fe599f5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.599358655Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.599401466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 6a3e1632-3f67-4c67-a09b-2ee7c055f6de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.619234804Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.619287075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5dc3d2e8-7d3d-4338-9928-132828621316, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 45f571be-936e-48ca-b02e-6bbc2b2f1974, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.63839326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.63843467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 45f571be-936e-48ca-b02e-6bbc2b2f1974, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.773837652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.773888862Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 709ff80f-750f-45c6-b668-97969f356dc0, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.970274347Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:12.970328288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 709ff80f-750f-45c6-b668-97969f356dc0, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b880f4d-eaa0-4b77-919b-817d6a980247, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.079650269Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.07969589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5b880f4d-eaa0-4b77-919b-817d6a980247, dataset_name: DatasetName { pool_name: ZpoolName { id: 05effe78-0c08-40b4-916b-cde344744ec3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa3cf0ac-1c68-4e1c-a127-490862a19932, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.186598423Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.186644914Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: aa3cf0ac-1c68-4e1c-a127-490862a19932, dataset_name: DatasetName { pool_name: ZpoolName { id: 866d818f-82bb-444c-992a-1351dec91efa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a83427f-50d0-47ad-8765-fad7f4c19873, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.320399683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.320446074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 5a83427f-50d0-47ad-8765-fad7f4c19873, dataset_name: DatasetName { pool_name: ZpoolName { id: 4fbbfff0-e175-4404-8cd9-acfc358b7140 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f389808b-a883-46ea-a396-af6431b5b6e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.440041545Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.440089746Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f389808b-a883-46ea-a396-af6431b5b6e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 6eaf0007-0aa9-44b6-8bac-b01569f4c6a3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23eb92fe-33fd-4e95-a819-b3349525e67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.566821148Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.566867919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 23eb92fe-33fd-4e95-a819-b3349525e67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 775439b8-d6ea-4b72-9708-a30c4c7c6d6f (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d98c3f2a-4c9c-4655-9407-3839f53cf851, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.68441357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.684472052Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: d98c3f2a-4c9c-4655-9407-3839f53cf851, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f77dae-534d-4b73-b38a-890d525d3747 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57f24d92-52ba-4db7-a1b1-c2a25e015a03, dataset_name: DatasetName { pool_name: ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.788109781Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.788149922Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 57f24d92-52ba-4db7-a1b1-c2a25e015a03, dataset_name: DatasetName { pool_name: ZpoolName { id: 1b55399c-9dbb-4d32-bd51-426a7996ea24 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9becf607-a424-4af6-bd2e-178c196b8348, dataset_name: DatasetName { pool_name: ZpoolName { id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.901858397Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:13.901899268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9becf607-a424-4af6-bd2e-178c196b8348, dataset_name: DatasetName { pool_name: ZpoolName { id: aad99d77-5c0c-4ae7-8cc1-ced403d6b98a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3da8a36e-1727-4d21-9b85-8be498e95b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.150465775Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.150533597Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 3da8a36e-1727-4d21-9b85-8be498e95b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: f4912882-4c2a-4c0d-b986-5cbff960587e (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.282629583Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.282695424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.282910809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.283046691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.404409788Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.404560681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.416744799Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.41830585Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.490385611Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.492112325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.492194247Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.492200007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.492203367Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.492206647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.639813477Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708278858Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708482462Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708503162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708518202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708527453Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708536903Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708545483Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708555173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.708658805Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.714629062Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.719292133Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.723809092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.728925952Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.733962091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.739378297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.883987777Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.890683188Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.895487502Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.901874038Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.906768634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.912098978Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.916837441Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.92190413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.927072351Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.932190171Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.937305111Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.943200537Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.948555622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.953795364Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.996648163Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:14.997240525Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.002719612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.007930064Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.012729158Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.017377629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.02253537Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.027325754Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.068870517Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.102784122Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.13693631Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.139220455Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.139229315Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.173811472Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 134, 65]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.173852713Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.219325134Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.22120102Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.221215731Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.272945013Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 132, 224]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.272974684Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.274407212Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.274420142Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.32894139Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 235, 245]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.328982621Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.332832796Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.332851897Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.440065356Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 217, 255]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.440099746Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"}
{"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.440417573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.440432863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.441853091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.441865771Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.493698196Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 188, 52]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.493731517Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"}
{"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.494048923Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.495734556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.541174356Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.541268098Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.541504242Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.541519312Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.541809558Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.541824488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.542022452Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.542031533Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.542209846Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.545384978Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.548674572Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.548693633Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.548703033Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.55213443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.552156011Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.733714766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.734038312Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.810735264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.831934478Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.855353947Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.876024131Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.893199118Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.90709359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.922538972Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.937992525Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:15.954683772Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService23'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::18'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_1b55399c-9dbb-4d32-bd51-426a7996ea24/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::18'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='6a3faa12-4d1f-435b-b060-46272624f3fb'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.232451311Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.232657635Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService16'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::16'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_775439b8-d6ea-4b72-9708-a30c4c7c6d6f/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::16'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='9a3e84dc-e0c8-4d3e-9c8f-1b5a823c1471'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.241104021Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService21'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::19'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_aad99d77-5c0c-4ae7-8cc1-ced403d6b98a/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::19'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='5b382486-72e3-4c57-83d3-2d93598fd8a6'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.241306945Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.241355715Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.248980775Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService19'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::f'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::f'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.256122315Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.256333159Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService14'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::17'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_e4f77dae-534d-4b73-b38a-890d525d3747/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::17'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='60622c6b-cb7b-4511-b42d-0e97a87c6a92'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.276886001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.277047574Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService12'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::12'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_05effe78-0c08-40b4-916b-cde344744ec3/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::12'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='e24813ac-ee27-4b93-942d-ffa2b0053022'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.286119712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.286278555Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService15'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::15'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_6eaf0007-0aa9-44b6-8bac-b01569f4c6a3/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::15'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='e3fe6845-0be5-4cb9-a51f-9e0cb2d4a4de'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.316687511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.316883734Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService13'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::14'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_4fbbfff0-e175-4404-8cd9-acfc358b7140/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::14'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='2e0b2dcb-d65c-40bb-8422-47dd39ef82d7'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.32484567Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.325038524Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService17'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::10'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::10'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.407908197Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.4080927Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService22'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1a'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_f4912882-4c2a-4c0d-b986-5cbff960587e/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::1a'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='816f8166-ed77-4cfe-82cd-81b0533719ca'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.434796253Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:17.434969447Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 46.816698ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:17.968840919Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 55.306726ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:17.978001599Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 59.343624ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:17.985596218Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 74.441807ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:17.994739587Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 36.737529ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.003493918Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 41.361971ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.013480144Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 38.788008ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.022709444Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 62.583881ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.031212381Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 65.855994ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.05974433Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 65.992149ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.128499306Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 92.683917ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.143927438Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 73.330994ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.164197835Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 107.856286ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.183929201Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 133.988223ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.204422842Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 56.846848ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.224270111Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 130.388026ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.241477838Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 145.987516ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.261334167Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 64.879527ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.279857999Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 125.86292ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.298132397Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 227.541131ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.318646769Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 74.662678ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.338572589Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 130.755325ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.356256145Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 112.927229ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.375440781Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 253.488843ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.456474918Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 176.142147ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.504729763Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 150.548514ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.532358924Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 154.880614ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:18.572203724Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 259.251154ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.172532248Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 130.656486ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.239883917Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 369.990918ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.267182652Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 169.904625ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.284892099Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 356.966387ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.300684238Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 312.088385ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.32325325Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 525.410368ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.359781895Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 522.532045ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.435737292Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 269.685854ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.448461231Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 538.561535ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.461719361Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 344.845282ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.475783636Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 570.855057ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.490059586Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 588.827349ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.503931047Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.089299362s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.654756171Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 564.127574ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.684911751Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 684.75022ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.715759785Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 795.957415ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.751586097Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.027870295s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.84169746Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 729.602273ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.901697075Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 663.056813ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:19.974662384Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 908.537222ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.01789785Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 421.065455ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.078926625Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 954.363829ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.111014483Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 649.718677ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.267424666Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 517.271149ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.416134408Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.295878595s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.515705768Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.123765931s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.564728738Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.090132658s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.647330865Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 687.97612ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.658225298Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 871.961011ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.759629604Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.032669234s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.884424367Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 990.52918ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.930681563Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.2715803s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.94231765Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 732.530321ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:20.952977189Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.296915109s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.079608588Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.326780213s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.445226558Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.428080366s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.647240943Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.111849814s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.70124001Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.085245366s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.71503777Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 798.045154ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.752258079Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.048101804s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.825695827Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.008753356s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.930273405Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 557.324518ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:21.94125166Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService10'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::d'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value='776ef919-ac0a-4b14-9dfe-deac75d16696'/>\n        <propval type=\"astring\" name=\"address\" value='[fd00:1122:3344:101::d]:12223'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:22.083623968Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:22.084154738Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService18'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::11'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::11'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:22.089818499Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:22.089954091Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService20'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::13'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_866d818f-82bb-444c-992a-1351dec91efa/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::13'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='0de3f60b-6c72-4697-9744-e3096340231e'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:22.132442403Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:22.132559976Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 881.387514ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.224053887Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 58.130941ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.246701991Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 55.117997ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.258661925Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 33.628622ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.282387519Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 109.756943ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.313234954Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 84.134014ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.321485275Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 56.793724ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.329617704Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 854.054417ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.386323225Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 197.213148ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.393083207Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 234.249542ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.41726168Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 288.819932ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.432605321Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.290480862s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.509143639Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.014896419s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.560206159Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 582.086866ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.600026869Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 427.713004ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.661096855Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 301.016456ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.731212048Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.192368442s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.781687816Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.030465654s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.809211685Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.148512474s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.82223709Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.342263775s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:22.884174792Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.249272872s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.036017745Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.051938444s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.059495575Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 960.250499ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.179929503Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 405.014718ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.19357648Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.170418475s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.208398241Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 630.636346ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.220757843Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 586.738268ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.251972154Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 512.080852ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.587480893Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.135037394s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.618895499Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.002254373s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.813129172Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 694.413627ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.852117755Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.298578566s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.862493208Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.013209966s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.872721409Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.47085188s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.982380586Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.273251833s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:23.99232659Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 797.864455ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.112486933Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.487813477s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.123105061Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.369350518s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.151497467Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.441149841s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.237091153Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.436470204s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.296401694Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.220353576s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.391020297Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService11'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::e'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::e'/>\n        <propval type=\"astring\" name=\"listen_port\" value='8123'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"clickhouse_address\" value='[fd00:1122:3344:101::e]:8123'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::e]:8888'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:24.437201791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:24.437395645Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 854.60059ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.559477345Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 41.9639ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.654332913Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 64.946891ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.705118231Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 790.725381ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.764574221Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 268.653934ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.777903082Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.271937257s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.824560115Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 961.275855ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.896050705Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 931.617668ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:24.920003934Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService26'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::c'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte3'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.7'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:25.043041483Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:25.043121995Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 398.654124ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.079933586Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.188779023s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.172269074Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 39.280903ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.201916634Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 114.729976ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.248924035Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.399584977s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.275163668Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 230.261551ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.375445242Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.173469952s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.423011193Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.031089492s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.462261102Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 677.205211ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.48977664Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 623.988741ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.530305914Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 928.229648ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.563144177Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 223.981667ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.616021312Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 660.284263ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.624967128Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.265419441s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.63377102Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.080189139s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.686930121Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.058446855s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.741871107Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 1.105325745s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.848993993Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.290259779s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.861275784Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.037993654s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:25.869583907Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.417651145s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.105656179Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 552.8967ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.162719876Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 1.285648097s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.174537368Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.316655909s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.292698862Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.430636771s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.368815142Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.343259963s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.499016801Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.372978237s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.50558317Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.243954378s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.604078209Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 783.931295ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.683197078Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 802.429831ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.737373069Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.466729611s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.833826277Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.250616229s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:26.971802118Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 673.753107ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.017700267Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 622.999602ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.059271291Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.133693141s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.080607309Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.092473456s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.165538392Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 1.244053515s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.471121746Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.4603862s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.481214393Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 774.554004ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.533902085Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.060318818s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.549298986Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 839.752662ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.619292347Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.481622778s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.691564332Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 721.029247ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.70065718Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.115102146s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.809518001Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 787.687396ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.852280899Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService28'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::9'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte5'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.1.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.1.6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::9]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='172.30.1.6:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:27.852597505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.328601104s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.860621862Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:27.860882307Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 802.342418ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:27.888850575Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 45.62892ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.094541142Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 67.131458ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.148974698Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 993.622245ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.223266363Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 103.631148ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.230711959Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 693.094333ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.242686973Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.477205389s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.268794974Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 945.095347ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.308762407Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 814.925387ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.316624771Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 209.981133ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.344238912Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 1.250330979s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.432045681Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.467535277s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.467886923Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService25'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte2'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:28.520189117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:28.520275789Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.05822442s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.564135577Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService24'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::a'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte1'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:28.589613156Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:28.589701428Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.275503307s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.620300887Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.278335303s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.649943658Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 71.579584ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.677832474Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 661.789701ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.699814634Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 691.771755ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.724346725Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 62.446386ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.731306651Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 79.764238ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.757168457Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 124.859139ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.803034526Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 164.864621ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.847514996Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 870.830216ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.934202533Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 189.649332ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.943845102Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.001578033s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.951090744Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.382248918s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:28.958288345Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 399.80287ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.023495521Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.39744566s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.142615464Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 273.336198ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.150430437Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.295276915s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.181521786Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 507.695939ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.197941607Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 805.580596ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.227011696Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 773.050091ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.263130434Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 516.368702ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.37120182Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 787.824721ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.425752598Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 427.32072ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.434643692Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 763.168602ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.44272044Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.198365311s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.632786142Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 691.10591ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.690790228Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 816.327819ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.713562723Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 872.999694ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.753586597Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.481297446s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.81243391Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 789.20595ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.87021342Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 616.905124ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.904336478Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.309904817s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.913656311Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 538.001339ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.939065038Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 705.693102ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:29.9488738Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.408486162s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.125962978Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.336759285s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.245828125Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.310162044s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.279003914Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 797.587758ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.296096179Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 996.309165ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.451562443Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 670.039673ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.514234891Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.331899981s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.564647518Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.472710861s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.588012285Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 650.819051ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.600192264Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 926.685082ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.612505805Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 575.485578ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.623255335Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.371066546s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.634873783Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.012138009s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.650463328Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.039800083s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.665520883Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 1.401338815s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.674518099Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.263078595s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:30.840686922Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 978.925623ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.104185521Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 786.026325ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.193730135Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.264320338s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.208392492Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 791.956835ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.232908702Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 731.080144ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.260128195Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.474581323s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.302693118Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 746.776577ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.457389137Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 788.440123ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.543183687Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 917.23303ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.551314517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.238785755s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.589707038Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.282520363s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.597374939Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 802.392802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.670222955Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 653.674348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.71288487Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 665.270542ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.905844208Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 1.166652413s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.989083908Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.149171317s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:31.999804398Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.194810676s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.014271671Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.26809989s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.032889055Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.072268994s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.070610734Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 1.151698218s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.084982495Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 518.847861ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.092719877Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.352098494s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.112574096Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 1.054341353s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.214032352Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.130928682s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.342535438Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.308834416s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.375367781Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.070356946s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.479164324Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 695.740134ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.487875704Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 872.600447ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.49585382Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 709.368213ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.580916386Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 1.111869242s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.622102073Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 564.889696ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.786273817Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.32840225s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.837285125Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.189501645s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:32.88808754Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 570.450773ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.152063509Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 723.138282ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.160543255Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 769.569366ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.169597652Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 945.668447ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.193315477Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.138212207s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.217699974Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 1.44944176s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.246574019Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 763.524781ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.276690949Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 523.557746ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.299102308Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 940.833916ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.309771917Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.164609905s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.3604758Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.486055919s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.376098216Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 684.295062ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.474230077Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 517.145203ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.481916388Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 717.95703ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.559215891Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.166206225s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.694243695Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.415705307s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.733558885Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 899.247461ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.742205614Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 693.795283ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.833502311Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 534.609113ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.894884263Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 1.131396834s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:33.951085124Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.386271929s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.011265502Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 975.91013ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.053724563Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 951.876098ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.091453242Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 834.171582ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.1515836Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.292167895s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.172130902Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.409421229s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.183844631Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.278468532s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.263554002Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 980.276542ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.291780635Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.111478083s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.380129495Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.037652572s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.447153967Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.423324861s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.545741328Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 605.295821ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.562682549Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService27'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::8'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte4'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.1.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.1.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::8]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='172.30.1.5:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:34.563121498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:34.563348152Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 1.040103625s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.659615687Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 801.74006ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.716710185Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 523.464545ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.88191066Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 869.636174ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.911086881Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 35.577354ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.929696566Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 104.72522ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:34.999138525Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.031831298s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.027438629Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 960.424096ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.064197739Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 921.532271ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.097141914Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 1.438010887s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.116366071Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 257.498791ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.136206389Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.126684049s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.170078213Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.180109364s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.198162383Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 718.735835ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.313125183Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 499.157796ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.421487745Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 576.72574ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.434289376Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 620.592713ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.449774489Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.353969226s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.496087786Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 966.932561ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.541802651Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.111443217s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.557187573Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 878.141161ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.579989379Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.062386366s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.599748546Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.451611893s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.620719446Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 1.373465479s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.716498712Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 994.540288ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.809151076Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 1.005485038s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.939392145Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.365459921s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:35.987366925Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.184172072s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.035529538Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 1.0040339s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.067223959Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 968.985676ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.088565597Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 743.862578ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.106351355Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.33134984s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.121716735Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 554.401009ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.135845252Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.300857143s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.308696877Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 809.255616ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.389625941Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 1.428451816s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.468913964Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 998.006732ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.50857355Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 957.709301ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.564538096Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.399847802s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.674350336Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.263927929s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.684924984Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.194122659s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.701171202Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.212359833s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.814331577Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 645.248562ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.859820427Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 531.830802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.868403866Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 1.257659454s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:36.955199785Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 845.881106ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.070615815Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 725.564604ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.081519748Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.379725619s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.09026693Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 1.304450851s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.100590352Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.099151132s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.21182684Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 827.435009ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.233141327Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.246641122s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.365859146Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 904.173439ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.411814276Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 605.043174ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.463947117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 583.285851ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.515804922Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 734.813819ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.524977602Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 689.95374ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.536268233Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 837.363616ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.621524452Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 555.572543ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.819863986Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 1.181497297s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.909650533Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 844.949517ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.9242804Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 991.951444ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.936719593Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 785.643425ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:37.962651581Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.054281508s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.043771359Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.400797075s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.072613284Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.104683472s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.084229092Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 892.105179ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.094227887Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 1.492145196s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.110466585Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 790.797962ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.225168911Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 714.85077ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.236184527Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 569.370987ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.270521369Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 1.451487821s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.321562859Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 603.998658ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.331668517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 823.480687ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.385884548Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 1.237418176s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.418874034Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 796.833059ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.470689909Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.465850897s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.481509271Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 521.25147ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.625558361Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 1.380093061s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.765587363Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 1.126848613s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.783735529Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 805.277329ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.855220517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 585.391726ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.943296312Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 929.686025ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.957908718Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 1.335993963s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:38.967950595Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.002000612s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.002169575Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 931.978981ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.030175913Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 968.519692ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.107648Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 1.078352996s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.121038312Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691\") failed: Property not found. retry in 1.283397775s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.162003564Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 859.829775ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.203976616Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 1.394577475s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.221752664Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.486632861s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.286241747Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696\") failed: Property not found. retry in 1.422946245s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.489992967Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247\") failed: Property not found. retry in 1.337393823s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.547924701Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 880.624123ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.622511901Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5\") failed: Property not found. retry in 1.280951137s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.679382945Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 538.634473ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.695787916Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 564.125854ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.789038012Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 637.460165ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.91202672Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9\") failed: Property not found. retry in 1.495738969s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.922095157Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 556.226463ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.96005544Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 1.118577774s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:39.991123258Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3\") failed: Property not found. retry in 1.252190516s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.025268537Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873\") failed: Property not found. retry in 1.049137211s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.08156036Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2\") failed: Property not found. retry in 922.570576ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.091167797Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c\") failed: Property not found. retry in 847.209659ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.165192737Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a\") failed: Property not found. retry in 707.070354ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.214548493Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851\") failed: Property not found. retry in 1.004740284s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.252286502Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e\") failed: Property not found. retry in 727.118689ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.323848374Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9\") failed: Property not found. retry in 586.244787ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.370236912Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9\") failed: Property not found. retry in 639.228228ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.524636585Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348\") failed: Property not found. retry in 1.31170411s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.547808079Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03\") failed: Property not found. retry in 1.027244931s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.575346938Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0\") failed: Property not found. retry in 1.45098224s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.638061866Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932\") failed: Property not found. retry in 1.127826397s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:40.79503542Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 932.506691ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:41.332434173Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 663.224774ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:42.326273362Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:42.508381118Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35625"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:42.51254522Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"9ab68e08-5ea9-4d5b-9e66-68ffae55d766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35625","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974\") failed: Property not found. retry in 1.283719925s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:43.013247923Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334592495Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_oximeter_776ef919-ac0a-4b14-9dfe-deac75d16696"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334658926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_a66efbaf-5f9a-40ef-9401-e26ed785dbe9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334667917Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_external_dns_49d104a1-1335-4930-9b9e-1ec4a3ca16f9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334683867Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_d36d5399-1e8d-42d4-b1c6-82ab39a196e5"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334692297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_clickhouse_709ff80f-750f-45c6-b668-97969f356dc0"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334700277Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_5b880f4d-eaa0-4b77-919b-817d6a980247"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334707777Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_5a83427f-50d0-47ad-8765-fad7f4c19873"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334716597Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_d98c3f2a-4c9c-4655-9407-3839f53cf851"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334723778Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_f389808b-a883-46ea-a396-af6431b5b6e3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334732978Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_23eb92fe-33fd-4e95-a819-b3349525e67a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334743398Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_nexus_a752201f-8e19-45ac-9901-65317e43fc2e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334750808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_45d4b7eb-c738-4794-9125-01d0ae07feb2"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334758448Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_bbf9c6ad-fa23-470f-b608-67105cf521a9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334763758Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_pantry_4c2f62ac-cb40-467c-ba9f-de7aa05f3691"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334771508Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_aa3cf0ac-1c68-4e1c-a127-490862a19932"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334783009Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_9becf607-a424-4af6-bd2e-178c196b8348"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334792659Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_3da8a36e-1727-4d21-9b85-8be498e95b8c"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334799979Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_crucible_57f24d92-52ba-4db7-a1b1-c2a25e015a03"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.334806319Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3250","zone":"oxz_external_dns_45f571be-936e-48ca-b02e-6bbc2b2f1974"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.33482902Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.336086004Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.33792272Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"757b4cd9-d074-49c0-9f45-b5efba9434dd","remote_addr":"[fd00:1122:3344:101::1]:65026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":31863706,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.342432558Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1286"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.342456009Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.342713174Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.346797594Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:687"}
{"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.346845165Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:700"}
{"msg":"rack_network_config: RackNetworkConfigV2 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.2.174,\n    infra_ip_last: 10.151.2.174,\n    ports: [\n        PortConfigV2 {\n            addresses: [\n                UplinkAddressConfig {\n                    address: V4(\n                        Ipv4Net {\n                            addr: 10.151.2.174,\n                            width: 24,\n                        },\n                    ),\n                    vlan_id: None,\n                },\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            lldp: None,\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Net {\n                            addr: 0.0.0.0,\n                            width: 0,\n                        },\n                    ),\n                    local_pref: None,\n                    nexthop: 10.151.2.100,\n                    vlan_id: None,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Net {\n        addr: fd00:1122:3344:100::,\n        width: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.379385602Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:867"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.379989594Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.455507292Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.62901205Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.81132251Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:44.999592705Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:45.158023267Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:45.327365563Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:45.526922811Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:45.774355296Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:46.035796745Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:46.702289436Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:47.460368869Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:47.998064366Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:48.819604633Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:49.99927209Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:51.668103616Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:52.994417176Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:18:54.086298575Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:54.086325245Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:54.480403742Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:54.485708146Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:18:55.089472078Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:18:56.092797163Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:56.479595837Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:56.49506397Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:18:57.096935454Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:57.92103329Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:18:58.100673407Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:18:58.479341822Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:18:59.104105955Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0c368d89-20a1-4c09-9eea-581bb60b3770/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:18:59.689133901Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:19:00.107310238Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:19:00.479407004Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b9c40332-a231-42db-adf0-b267d663a311): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-29T15:19:01.114778674Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1ffcd8ea-e615-45b9-8415-54f197693d3a\", \"content-length\": \"173\", \"date\": \"Thu, 29 Aug 2024 15:19:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"0c368d89-20a1-4c09-9eea-581bb60b3770\\\"\", request_id: \"1ffcd8ea-e615-45b9-8415-54f197693d3a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:03.12018426Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.57776257Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.667498067Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33841"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.668042918Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.746408822Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"243235e1-62f9-4de7-99d7-a48947be1256","remote_addr":"[fd00:1122:3344:101::a]:33841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":78819,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.746643317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.746710288Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.747468933Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:04.747687867Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"b995e923-f7aa-4991-8ef0-61249f2945b2","remote_addr":"[fd00:1122:3344:101::a]:33841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550050839Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41070"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550370525Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550422886Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550458057Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550482148Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550493958Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550505778Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550517138Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.550548479Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"dfb5d0d9-7639-4919-bb9c-cec0f2f16e0f","remote_addr":"[fd00:1122:3344:101::a]:41070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":461,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.552841594Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:939"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.907813134Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47586"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:09.90864638Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.004820103Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"19631bfc-8ef1-4cc8-8253-6c3297b0c25a","remote_addr":"[fd00:1122:3344:101::a]:47586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96905,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.00515191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.005227211Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.005553987Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.006092348Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"8334647d-1d5b-49f6-80ff-1388bff3c277","remote_addr":"[fd00:1122:3344:101::a]:47586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1030,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.125744051Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42797"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126659409Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.12672362Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126757891Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126771701Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126790981Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126796832Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126803412Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.126833992Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c7c6d734-5163-413a-b7ee-4e77785b8c61","remote_addr":"[fd00:1122:3344:101::a]:42797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":996,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.297903652Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49176"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298124916Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298175957Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298202378Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298227738Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298236018Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298244979Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298250699Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.298279209Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b498b695-e8ce-4cd4-9c4d-dbe6915912b8","remote_addr":"[fd00:1122:3344:101::c]:49176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":297,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.34838279Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60529"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348576814Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348614715Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348625815Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348636035Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348644936Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348654356Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348661896Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:10.348670896Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d240b26f-713f-44c4-bfbe-f689a03e0516","remote_addr":"[fd00:1122:3344:101::b]:60529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":199,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:12.551288324Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64403"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:12.551473848Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1f27779f-7d3e-4bba-8ec1-ae6901343175","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64403","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":31,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:18.47284886Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37831"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:18.472981523Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bd38a7d8-f596-4990-9b98-466226002c98","remote_addr":"[fd00:1122:3344:101::a]:37831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":131,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:18.551519221Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bc62cc65-806f-4d54-a23e-38a4e600bf0c","remote_addr":"[fd00:1122:3344:101::a]:37831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":708,"response_code":"204"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:29.435225546Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.77954617Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36529"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.779700393Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1bb68e8e-508e-4691-a8f7-b2cfde10d094","remote_addr":"[fd00:1122:3344:101::a]:36529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.804386277Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ae2f2eb1-0fa4-4eba-bfc1-a7138b22c71a","remote_addr":"[fd00:1122:3344:101::a]:36529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.857439695Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.857557167Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"efb51599-bce4-4dac-959d-72db079bc5f9","remote_addr":"[fd00:1122:3344:101::b]:53958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.878067979Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e137a4ca-d34e-45f1-8dc9-e05aff71dcb9","remote_addr":"[fd00:1122:3344:101::b]:53958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":203,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.914267198Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58460"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.915356609Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c47ab61a-a5a6-47a1-8afb-51bada1cc923","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1052,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.920729704Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37885"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.920967989Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8d544c33-b134-495e-9824-ebb3f08357bb","remote_addr":"[fd00:1122:3344:101::a]:37885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":186,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.937458032Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4ae60ceb-c3fd-4dcd-b753-d7d0090dfda2","remote_addr":"[fd00:1122:3344:101::c]:58460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":157,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.973548198Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56074"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:33.973694841Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.026490895Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.089140962Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"65485761-657c-49f0-8dfa-d9d653ebf39b","remote_addr":"[fd00:1122:3344:101::a]:56074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115540,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.089372516Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39472"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.089518319Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59739"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.089659112Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"979baee9-e9ec-4c3e-8d96-6769dde4099d","remote_addr":"[fd00:1122:3344:101::c]:39472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":235,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.089791715Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"191582b6-86c2-4629-b352-82b937906e10","remote_addr":"[fd00:1122:3344:101::a]:56074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":302,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.09060175Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"15c78246-2327-4a90-aebd-177a35abd812","remote_addr":"[fd00:1122:3344:101::b]:59739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":983,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.123986034Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36790"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.124729299Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.164735342Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.225622904Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fc8ea60d-2c34-462b-a54f-acf6f4f69e9e","remote_addr":"[fd00:1122:3344:101::b]:36790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101537,"response_code":"204"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.226009482Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.226041402Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.227161754Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.22794469Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.22799001Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"24f99483-5596-4076-810f-ceb97b060c49","remote_addr":"[fd00:1122:3344:101::a]:56074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2200,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.228030301Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"857b4a37-497e-46c7-b7a2-e47c32afd842","remote_addr":"[fd00:1122:3344:101::b]:36790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1929,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.281109981Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44066"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.281250904Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.315730179Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.369821798Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2eb6e8aa-c9d7-47f0-a027-6de7958c5892","remote_addr":"[fd00:1122:3344:101::c]:44066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88627,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:34.370338688Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9e6f5a31-788f-430c-94f0-f76be59b51e1","remote_addr":"[fd00:1122:3344:101::c]:44066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":212,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:42.5854214Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35102"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:19:42.585501121Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"fe8ee4fb-8a15-4d4c-b22d-0f84c7cbb787","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35102","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:03.965365384Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48971"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:03.96667805Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b30b99ba-1a6a-4583-9d6a-3d32105de686","remote_addr":"[fd00:1122:3344:101::b]:48971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1260,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:03.98916082Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46912"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:03.989263012Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e0197960-7107-4fd0-b4cd-8ff11134592e","remote_addr":"[fd00:1122:3344:101::a]:46912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.002080963Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b800aa81-9ea6-4b46-a1ef-dcd9716fbafa","remote_addr":"[fd00:1122:3344:101::a]:46912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":138,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.073046743Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57276"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.073220086Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"9e1fd0f4-c966-4d80-99d2-51eb3c99a41e","remote_addr":"[fd00:1122:3344:101::c]:57276","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":154,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.091692428Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35579"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.09180435Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"90882e64-e7f8-470c-bb70-bd39ad9cf4ba","remote_addr":"[fd00:1122:3344:101::c]:35579","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.103729063Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"93202793-60af-46fb-95eb-69dfd33027e4","remote_addr":"[fd00:1122:3344:101::c]:35579","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":969,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.15768709Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45572"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.157853933Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"6f02b2df-9ef4-442c-b649-f10efedb42f7","remote_addr":"[fd00:1122:3344:101::b]:45572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.167105604Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ae15e966-e872-48db-8c6a-8c8d1fdc7b33","remote_addr":"[fd00:1122:3344:101::b]:48971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":80,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.224196632Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62216"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.224352965Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"4aba0bcc-dd7f-4770-83ad-af288ed452a4","remote_addr":"[fd00:1122:3344:101::a]:62216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":84,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.330067055Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54178"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.330175748Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.366608461Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.419528507Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"14998f7b-4bf1-4ab7-b737-b37e2462e94f","remote_addr":"[fd00:1122:3344:101::a]:54178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89407,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.419834133Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63958"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.419905614Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57943"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.420145709Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"06242728-ca67-44fd-b062-117b6bf0672d","remote_addr":"[fd00:1122:3344:101::a]:54178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":313,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.420230041Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.459545641Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.517664239Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.51773089Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d5dc547b-7978-4c62-9f12-78dba0b4742d","remote_addr":"[fd00:1122:3344:101::b]:57943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97637,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.521619386Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3dcf6dda-8def-4553-949d-c4b894670b42","remote_addr":"[fd00:1122:3344:101::b]:57943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":700,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.555180453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.609741672Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ee563b79-657f-4775-b82a-3c9d7b3b5301","remote_addr":"[fd00:1122:3344:101::c]:63958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":189659,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:04.610117079Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b8fc0f5d-e4d4-47f2-92a5-96d8bf370b72","remote_addr":"[fd00:1122:3344:101::c]:63958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:12.30462645Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:51670"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:12.358811891Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"f6fd873e-67ca-4cd7-aeeb-d4cc1d56daea","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":54108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:12.618870433Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33843"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:12.618943974Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"29b07778-6d00-49fd-8127-205a9241d6ff","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33843","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.805519472Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61936"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.805658605Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"2bb9e2a7-fb48-4931-9170-9816a62cfb18","remote_addr":"[fd00:1122:3344:101::b]:61936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.82379752Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c7e8e9e5-b77a-4e7c-a236-7125e5caab63","remote_addr":"[fd00:1122:3344:101::b]:61936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":174,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.827970261Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33350"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.828133734Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"cbc726a1-cc38-4981-8d34-b3b0b6d11f8e","remote_addr":"[fd00:1122:3344:101::a]:33350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":53,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.842949124Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"16ed82be-159c-4640-9516-82c0b2ba72af","remote_addr":"[fd00:1122:3344:101::a]:33350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":128,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.913411444Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63264"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.913495596Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a8cf00d3-e642-4472-8d90-56b5154d2ec1","remote_addr":"[fd00:1122:3344:101::c]:63264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.923672095Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"24cc28cd-16d7-4a36-8539-1892e0667d22","remote_addr":"[fd00:1122:3344:101::c]:63264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.937807222Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40353"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.937988505Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"fc533a6e-3847-425c-94e8-0f937e368914","remote_addr":"[fd00:1122:3344:101::a]:40353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.965359312Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43224"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:33.965477224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.008579968Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.072607841Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"cc2cea85-7546-4084-9ab1-ec384ca3dab6","remote_addr":"[fd00:1122:3344:101::a]:43224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107172,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.072823516Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64289"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.072935058Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52263"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.073132972Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"83633b0f-a57c-4561-b708-543a136390fb","remote_addr":"[fd00:1122:3344:101::b]:64289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":234,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.073261754Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"336eecb4-b97d-4e11-995a-98fc48a2c3a8","remote_addr":"[fd00:1122:3344:101::a]:43224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":258,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.074157362Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.093860557Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39992"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.094203504Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"5a6b384e-193a-4784-bccd-ff55ec4a5db8","remote_addr":"[fd00:1122:3344:101::c]:39992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":257,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.115571263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.158611105Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55121"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.17669889Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.176788751Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4e25449c-a092-4682-a270-e32be622ccfd","remote_addr":"[fd00:1122:3344:101::c]:52263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103645,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.181046995Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2a5aeeb2-722d-44af-b613-862814cc851b","remote_addr":"[fd00:1122:3344:101::c]:52263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":730,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.213655113Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.268383145Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2ae10042-1b70-4afe-a41f-eebc6f971941","remote_addr":"[fd00:1122:3344:101::b]:55121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109663,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:34.268780273Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6f52f1b5-383b-40fb-8ce4-c1597694b99a","remote_addr":"[fd00:1122:3344:101::b]:55121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":136,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:42.310698196Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"aa54dceb-27f3-4b0b-9930-23572b875b01","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5992,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:42.653262304Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62561"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:20:42.653322115Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"9014572f-53a5-4114-84bc-7aba4b8529a5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62561","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:03.914456652Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49526"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:03.91590974Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3fc4b556-a07b-4b03-b48e-b33b3981fd73","remote_addr":"[fd00:1122:3344:101::a]:49526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1412,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:03.938059164Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"78563c1b-e7b7-4d3e-be86-1372f9e1c5f2","remote_addr":"[fd00:1122:3344:101::a]:49526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":185,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.010435291Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33076"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.010526553Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b20b595e-de80-49a5-897f-b4fec706ca04","remote_addr":"[fd00:1122:3344:101::b]:33076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.022312334Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"dd996f8e-2a3f-4a54-9735-2318e9121117","remote_addr":"[fd00:1122:3344:101::b]:33076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.139804344Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51677"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.139889006Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"41ccc955-ca4c-434c-938d-cb85ce86ffa6","remote_addr":"[fd00:1122:3344:101::c]:51677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.156777077Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bd0a2b5d-9c04-4653-9fdb-3075eeda723f","remote_addr":"[fd00:1122:3344:101::c]:51677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.175566395Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50020"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.175729988Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f915a380-51a6-41c3-86ee-3e97ac7b2265","remote_addr":"[fd00:1122:3344:101::b]:50020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":138,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.195169108Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60086"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.19526374Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.237154951Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.301698774Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2d25a6fc-b025-4403-af7f-1d467035f4eb","remote_addr":"[fd00:1122:3344:101::c]:60086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106472,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.301893388Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43074"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.301914989Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54740"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.302091042Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45516"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.302219575Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f8938658-902e-4cfa-a9fc-6b79c3c2da50","remote_addr":"[fd00:1122:3344:101::c]:54740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":257,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.302311997Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.302440139Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"825d08a7-2da8-4a22-9118-023ea6a092c3","remote_addr":"[fd00:1122:3344:101::c]:60086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":340,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.309023328Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d8b8ef74-c718-470f-a828-e2679d899242","remote_addr":"[fd00:1122:3344:101::a]:45516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6732,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.34589854Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.403137481Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"db908637-f14d-4c8d-87cf-e3163c575441","remote_addr":"[fd00:1122:3344:101::b]:43074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101019,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.403567369Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c102799d-71bb-4eb4-88cd-1be6f51f684a","remote_addr":"[fd00:1122:3344:101::b]:43074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.514005511Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36823"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.514101413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.560265017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.627864941Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9fba97a5-92cc-4828-a5b2-0be8119483f4","remote_addr":"[fd00:1122:3344:101::a]:36823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113797,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:04.628205858Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2219d0f3-f8fc-4c33-b33d-bece02264e94","remote_addr":"[fd00:1122:3344:101::a]:36823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:12.311148271Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"a8dc28bd-d61c-4293-8914-900813375209","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7133,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:12.685576462Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53934"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:12.686694624Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e4ce2ec8-097b-4841-9f07-181445a9a7c8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53934","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1072,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.864052832Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59565"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.864299927Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0d13db58-273a-4c30-80fa-298980b3c124","remote_addr":"[fd00:1122:3344:101::a]:59565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":213,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.938163153Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44889"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.938247445Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b4534ffc-cc4e-4ed9-963e-8e05a20d4dce","remote_addr":"[fd00:1122:3344:101::c]:44889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.952222099Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1f59a65d-1588-46c0-9f5b-13d4b3d699e8","remote_addr":"[fd00:1122:3344:101::c]:44889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":892,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.980456351Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47443"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.980566094Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6287997a-6d29-482d-862a-2c8ab1ffdf0b","remote_addr":"[fd00:1122:3344:101::a]:47443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":36,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:33.995855093Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1b629300-9572-403e-bed4-12267f789390","remote_addr":"[fd00:1122:3344:101::a]:47443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.009957959Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33401"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.010042551Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"36173a76-9e62-4eae-9f28-49d614f12c54","remote_addr":"[fd00:1122:3344:101::b]:33401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.02074726Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e21f7c25-0bab-43d2-a4d0-3dc8b6d300e7","remote_addr":"[fd00:1122:3344:101::b]:33401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":67,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.036909367Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35554"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.036990078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.076061594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.134879285Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a6ac29f4-7328-424c-8ef6-b23216890b3e","remote_addr":"[fd00:1122:3344:101::a]:35554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97916,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.135021688Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46652"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.135226312Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d9e252c0-545b-4047-a2a4-99801f44c630","remote_addr":"[fd00:1122:3344:101::c]:46652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":155,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.135396095Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"078b19cb-7fda-4de1-924b-e35f530ca5fb","remote_addr":"[fd00:1122:3344:101::a]:35554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.13716184Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38463"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.137525587Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"035e2918-0233-441a-a647-cfe7a674c0d4","remote_addr":"[fd00:1122:3344:101::b]:38463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":233,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.179508819Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63556"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.179669572Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.220338859Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.280573548Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1f3780cc-6cad-4843-b37b-3a210e59aa7e","remote_addr":"[fd00:1122:3344:101::c]:63556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100967,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.280714281Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55542"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.280826583Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.315475862Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.369169843Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5a8ccc4d-f8e4-4826-b7cb-f4e43f30ac6a","remote_addr":"[fd00:1122:3344:101::b]:55542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88388,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.369413768Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d2a14630-6157-4160-91e0-98ee2922b75f","remote_addr":"[fd00:1122:3344:101::c]:63556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:34.36956669Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d23c0a3b-7b61-4461-903c-47e5de285c05","remote_addr":"[fd00:1122:3344:101::b]:55542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:42.311203571Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"6647322e-d05d-4299-a198-79111773660c","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6248,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:42.719628698Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56815"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:21:42.7197258Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3d048e0d-b852-4211-9710-76366f08ae8d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56815","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":54,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.960189903Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33331"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.961473778Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"143b0364-6883-4c09-9c74-7c4a0440a31e","remote_addr":"[fd00:1122:3344:101::a]:33331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1248,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.976587094Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"af9b31e3-e2ed-411c-a761-55b9ef790d53","remote_addr":"[fd00:1122:3344:101::a]:33331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":132,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.998668246Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53378"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.998793789Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"92afc4e8-caf9-4971-a702-7e4e89e0f334","remote_addr":"[fd00:1122:3344:101::b]:53378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.999810078Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56636"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:03.99988382Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"84ec219c-1a76-4151-9f14-884acbeef2ca","remote_addr":"[fd00:1122:3344:101::c]:56636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.021713358Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d78ab234-dc86-424e-84b0-c4a91f769bfe","remote_addr":"[fd00:1122:3344:101::c]:56636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1514,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.064857172Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41729"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.065038696Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"57c0b321-7d6a-4823-a19d-15f1d5d71590","remote_addr":"[fd00:1122:3344:101::b]:41729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":149,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.075565332Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d9f88dee-19c4-4997-b607-b142203c55c9","remote_addr":"[fd00:1122:3344:101::b]:53378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.106805064Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64336"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.106953996Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"634b7803-2b74-421b-9d05-7f6629ee1588","remote_addr":"[fd00:1122:3344:101::a]:64336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.155242562Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43379"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.155337784Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.196888808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.254565167Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fb261689-e2fe-4854-b943-59530ae519eb","remote_addr":"[fd00:1122:3344:101::a]:43379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99265,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.254770641Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59617"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.254983565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.259194877Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ef1ef03b-e405-4251-affb-c2d39c8875a4","remote_addr":"[fd00:1122:3344:101::a]:43379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4288,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.285817299Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43916"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.286133435Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"058d6ebe-6257-4b5d-832f-d17922c942c4","remote_addr":"[fd00:1122:3344:101::c]:43916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":269,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.294849846Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.333823569Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48125"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.351662528Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.351725599Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7121a51e-830e-4cb7-9ab6-589b062fb643","remote_addr":"[fd00:1122:3344:101::b]:59617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96852,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.357254418Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d5d11273-1256-4fe2-91de-62e5b78e1560","remote_addr":"[fd00:1122:3344:101::b]:59617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":690,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.389733064Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.443946445Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e2921db5-e5d4-454a-bc0b-b404b48b5745","remote_addr":"[fd00:1122:3344:101::c]:48125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109981,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:04.444289732Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"98f47fc8-e781-43da-a93f-2c6bb6470ebb","remote_addr":"[fd00:1122:3344:101::c]:48125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:12.321222944Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"290dd813-2e0c-45b7-8ccf-4945ccd49958","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6626,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:12.754317045Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44985"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:12.754877245Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"baa32853-fbeb-4692-91f6-22488b508187","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44985","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":546,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.825604743Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47755"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.825782647Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"69be1e7a-7439-412e-9d19-9f544dbe8e26","remote_addr":"[fd00:1122:3344:101::a]:47755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":131,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.838650718Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c21aea3e-44bc-48d6-856a-e5ff13e46ee6","remote_addr":"[fd00:1122:3344:101::a]:47755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":178,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.839576986Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61269"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.839688459Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"809b8724-9ce6-49a1-b632-471c83638990","remote_addr":"[fd00:1122:3344:101::b]:61269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.849298897Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"029e4e34-a7d7-4178-a70b-661f2b1d7d2b","remote_addr":"[fd00:1122:3344:101::b]:61269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1164,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.895963801Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42365"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.896072623Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"4226f0b8-33be-4902-9b4d-9928453925db","remote_addr":"[fd00:1122:3344:101::c]:42365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.906202641Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b0c38feb-f9fa-4dd6-8c98-7a1550f999ba","remote_addr":"[fd00:1122:3344:101::c]:42365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":154,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.943625524Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41187"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.943878029Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"e79b696f-b01d-4f41-9a10-155e56fdf44c","remote_addr":"[fd00:1122:3344:101::b]:41187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":133,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.972764094Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59908"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.972967709Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"56f26639-caea-4542-b3ae-9a3cf8966a21","remote_addr":"[fd00:1122:3344:101::a]:59908","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.974402286Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34170"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:33.974493908Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.015869518Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.08387195Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"19027d6a-245e-45c2-9b2f-9e1c3cf624a8","remote_addr":"[fd00:1122:3344:101::a]:34170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109402,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.084094784Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46730"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.084191026Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43441"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.084516603Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d8882da7-5c02-453b-95d0-cfe01c72a1cc","remote_addr":"[fd00:1122:3344:101::a]:34170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":351,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.085035233Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.089038291Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40069"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.089411518Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"46282195-6269-41ad-9d2d-a7bafd24ff5e","remote_addr":"[fd00:1122:3344:101::c]:40069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":234,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.123490386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.179506043Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.179575714Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1b331eeb-b4ab-4c62-bf13-680942e317ed","remote_addr":"[fd00:1122:3344:101::c]:46730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95355,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.183536612Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8793814e-adb6-47d2-b806-76b9b4ac7780","remote_addr":"[fd00:1122:3344:101::c]:46730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":310,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.214024309Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.271880942Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"de8f7fe7-8b5e-451e-b463-483eab7412b8","remote_addr":"[fd00:1122:3344:101::b]:43441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":187496,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:34.27282542Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9c5863bb-d1d5-4ae8-b402-b3f951051246","remote_addr":"[fd00:1122:3344:101::b]:43441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":686,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:42.391087726Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"3bedaa53-4c87-4ad8-a72e-c1c73c989bba","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7212,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:42.787111861Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59699"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:22:42.787186053Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"86f82d6f-e9fb-4d28-8d13-8e5d5190b50e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59699","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.904134947Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53409"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.90427615Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c83103be-6029-4196-b20a-cb3e23a9c407","remote_addr":"[fd00:1122:3344:101::a]:53409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.9165573Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bfa54c6d-16dc-477f-be40-7dbefba5c4b0","remote_addr":"[fd00:1122:3344:101::a]:53409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.934692185Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51222"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.93596491Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"33d3b0a9-9ce8-4fa1-93f7-594f3e500c3c","remote_addr":"[fd00:1122:3344:101::b]:51222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1210,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.989923047Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56325"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.9901132Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"a8fd07b0-a95c-4422-a922-2fe1a8b62729","remote_addr":"[fd00:1122:3344:101::b]:56325","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:03.99723274Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"731b71a6-1465-4430-8875-4575bda175ca","remote_addr":"[fd00:1122:3344:101::b]:51222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.044547626Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48066"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.044650628Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.091579727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.155792665Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"034efbb7-6e1f-4ad9-8007-eea8116ed37e","remote_addr":"[fd00:1122:3344:101::b]:48066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111188,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.156104841Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41243"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.156142362Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56541"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.156197803Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53254"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.156209893Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"22fa1e5c-f782-4adf-a4fb-0a0839255993","remote_addr":"[fd00:1122:3344:101::b]:48066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.156436527Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.156530469Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8e2e4f9d-70c5-44ce-ac40-5e786e3a01f5","remote_addr":"[fd00:1122:3344:101::a]:56541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":316,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.160854704Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"619db22e-2cec-4898-ad07-370481373766","remote_addr":"[fd00:1122:3344:101::c]:53254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4481,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.176194914Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40893"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.176374538Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"70f5ce12-de7d-4532-86f1-03cd69b2c426","remote_addr":"[fd00:1122:3344:101::c]:40893","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.179346166Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"227b1076-ac59-4544-aec3-221eb8c5dfc5","remote_addr":"[fd00:1122:3344:101::c]:53254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.191957603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.218331089Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54022"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.249706334Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.249756804Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"17691271-02fb-4d60-9397-6a8a08e5a047","remote_addr":"[fd00:1122:3344:101::a]:41243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93372,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.253138571Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"cd6edf79-a71d-44f8-8c0c-aa683b5f8cdf","remote_addr":"[fd00:1122:3344:101::a]:41243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":578,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.284586146Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.342109473Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"37de9358-d37f-44ed-9eec-3d95c9fa7b69","remote_addr":"[fd00:1122:3344:101::c]:54022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123611,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:04.342508311Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bde9c3c6-f65c-446c-83c9-fbd45e133e01","remote_addr":"[fd00:1122:3344:101::c]:54022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":147,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:12.390268469Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"5f5467dc-a92c-4e8f-8cad-757bd60c8ba4","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5666,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:12.82033436Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45021"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:12.820455992Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0f2def4e-8723-490e-9b35-8b1605186870","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45021","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.73126897Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61893"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.731428563Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"4b7e4dbf-8f6d-4500-9c48-a2ec9b0d703b","remote_addr":"[fd00:1122:3344:101::a]:61893","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.746797884Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f585760a-00c8-40f1-acd2-8325a0b61ec5","remote_addr":"[fd00:1122:3344:101::a]:61893","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.814080312Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44795"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.814190874Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9624e2b5-2770-4ed5-a5b5-84c502dbaa74","remote_addr":"[fd00:1122:3344:101::b]:44795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.826498135Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"82c7c72f-d8a9-4284-944a-f9d5e8c30594","remote_addr":"[fd00:1122:3344:101::b]:44795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.872497446Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63545"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.872591918Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"faae3eca-4c66-435a-8444-e31a3acb5c3d","remote_addr":"[fd00:1122:3344:101::c]:63545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.88243578Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6c6f2a84-d5bb-4a3d-b823-696c9bdf53d5","remote_addr":"[fd00:1122:3344:101::c]:63545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":121,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.909529461Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37905"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.909612012Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:33.948942502Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.011830694Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"82ed595f-6a34-42b7-b3f5-218fb16933bc","remote_addr":"[fd00:1122:3344:101::b]:37905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102251,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.012087489Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41655"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.01211573Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34559"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.01212589Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50823"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.01213379Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45770"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.012175161Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34912"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.012532168Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a7325f48-2e25-4215-a7d4-6509f9861275","remote_addr":"[fd00:1122:3344:101::b]:37905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":270,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.012608379Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"5f11d982-bda6-4402-999d-738fd2d9310b","remote_addr":"[fd00:1122:3344:101::b]:45770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":254,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.012617249Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"dc496e64-be31-46ad-8860-28e7319c481c","remote_addr":"[fd00:1122:3344:101::a]:34559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":339,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.013517967Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.013604119Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"51acc129-17bf-4f05-a6a2-7576a0984951","remote_addr":"[fd00:1122:3344:101::c]:50823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1214,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.05454072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.111046757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.111141258Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0df91bad-6909-425a-8a27-b536da6e0d00","remote_addr":"[fd00:1122:3344:101::c]:34912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98732,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.115042585Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0b3fc78c-7f17-4b00-a488-728ee58c44b4","remote_addr":"[fd00:1122:3344:101::c]:34912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":365,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.146368138Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.206905934Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"bf458bb5-b117-406c-974d-8dcfbc6148e5","remote_addr":"[fd00:1122:3344:101::a]:41655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":194457,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:34.207321962Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2cff1fa1-07bd-4d48-a4e8-253c3ac7aa1f","remote_addr":"[fd00:1122:3344:101::a]:41655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":175,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:42.394153653Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"0971aced-3e75-44c4-92fb-5453749bb53d","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7715,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:42.851921856Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48068"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:23:42.851993348Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"259e1a89-7394-4c1a-9533-1476c8bfa253","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48068","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:03.880030903Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38104"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:03.880209326Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6f96efe9-6f62-4932-b280-ac826455ce0c","remote_addr":"[fd00:1122:3344:101::b]:38104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:03.896481235Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ec1e1b4b-83ed-4e97-bc49-00acb92c41aa","remote_addr":"[fd00:1122:3344:101::b]:38104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1890,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.011205542Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54873"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.011295973Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3215403a-18f1-4cc7-a2e2-96887b761f93","remote_addr":"[fd00:1122:3344:101::c]:54873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.102408227Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60783"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.10251949Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"86e722df-8067-4dbc-b1db-68392545cc04","remote_addr":"[fd00:1122:3344:101::a]:60783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.117657776Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6290e15c-b52b-4c38-bf53-8c28c0dbd3b9","remote_addr":"[fd00:1122:3344:101::a]:60783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.146557532Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42113"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.146786456Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8f1b7196-c3fe-41d5-a7e5-c46b4f832c15","remote_addr":"[fd00:1122:3344:101::b]:42113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":193,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.175152382Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47598"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.175278174Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.231196179Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.301567477Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6935ec7a-30d2-43d1-8753-1a81cb9941d5","remote_addr":"[fd00:1122:3344:101::a]:47598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126351,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.302002845Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55757"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.302041646Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59998"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.302056457Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56873"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.302079227Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34494"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.302974035Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"aaab9fee-2c49-44ad-95e1-0af2f0812abb","remote_addr":"[fd00:1122:3344:101::c]:54873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1187,"response_code":"204"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.303043866Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.303139558Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d30d607a-8135-4e88-88cb-dcd7527b4597","remote_addr":"[fd00:1122:3344:101::a]:34494","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":958,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.303167358Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4363a001-04cb-4226-9cc3-df4f2d7307b2","remote_addr":"[fd00:1122:3344:101::a]:47598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1225,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.303191859Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"4454a32c-fd90-4714-be7d-3c1c33ddb942","remote_addr":"[fd00:1122:3344:101::c]:56873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":982,"response_code":"200"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.303225449Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.303293201Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.308558554Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.308699017Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.308714747Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.308751798Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.30884869Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.309115945Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9f206adb-c69c-48b3-8fc2-0c984cd29a02","remote_addr":"[fd00:1122:3344:101::a]:55757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6826,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.31092994Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58935"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.346739861Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.358320318Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34366"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.362944909Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363051871Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363084321Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363107332Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363132782Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363147083Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363192624Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.363267235Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"93ad66b1-b2ea-438c-90ad-2f7b95be44ba","remote_addr":"[fd00:1122:3344:101::b]:34366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4846,"response_code":"204"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.407771217Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.407837718Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"66a783d8-e61a-4c5a-a0f8-fe68c0cfb63e","remote_addr":"[fd00:1122:3344:101::b]:59998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105559,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.413852726Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7479bbeb-19f2-419d-91a1-26037ffba882","remote_addr":"[fd00:1122:3344:101::b]:59998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":769,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.448063656Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.508318805Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b040a7c6-5ffc-47c7-a114-f8feca9ff916","remote_addr":"[fd00:1122:3344:101::c]:58935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":197279,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.508734884Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9c29c873-fe9b-4f37-837c-98356f54c90e","remote_addr":"[fd00:1122:3344:101::c]:58935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":165,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552158214Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61647"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552340847Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9a1c332a-d0f3-4bf0-a446-7708135ac57c, kind: Service { id: 49d104a1-1335-4930-9b9e-1ec4a3ca16f9 }, name: Name(\"external-dns-49d104a1-1335-4930-9b9e-1ec4a3ca16f9\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 52])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ea86e1a2-b347-400f-a58e-d0e965a26ce3, kind: Service { id: 45f571be-936e-48ca-b02e-6bbc2b2f1974 }, name: Name(\"external-dns-45f571be-936e-48ca-b02e-6bbc2b2f1974\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 255])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 58c95a6c-0b6d-4e8e-9ca9-eb44bbbbd1be, kind: Service { id: a66efbaf-5f9a-40ef-9401-e26ed785dbe9 }, name: Name(\"nexus-a66efbaf-5f9a-40ef-9401-e26ed785dbe9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 65])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 65a401da-ec12-42f3-877e-1b2d9a2cf041, kind: Service { id: d36d5399-1e8d-42d4-b1c6-82ab39a196e5 }, name: Name(\"nexus-d36d5399-1e8d-42d4-b1c6-82ab39a196e5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 132, 224])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8f376814-87ad-4ea2-9c55-936c27a65b97, kind: Service { id: a752201f-8e19-45ac-9901-65317e43fc2e }, name: Name(\"nexus-a752201f-8e19-45ac-9901-65317e43fc2e\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 235, 245])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552387268Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552406979Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552414909Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552427009Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552434219Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.552441009Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:04.5524561Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7bac4ed7-5094-4a5f-a0f1-fb6d2a7d9de7","remote_addr":"[fd00:1122:3344:101::c]:61647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":233,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:12.390297988Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"6c918bb0-6e25-48fe-bf68-dffec1753f21","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6313,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:12.886209647Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33341"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:12.886282459Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"4fc4ad8a-66a7-4647-9da4-dd357c67b340","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33341","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:15.103697967Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:15.103785679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:15.146148858Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999578s","next_cleanup":"Instant { tv_sec: 21373, tv_nsec: 890481498 }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:29.435377896Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.718867288Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.720526061Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"34fc5c7f-602a-43ec-96a3-6edb39603896","remote_addr":"[fd00:1122:3344:101::a]:34038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1622,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.740762687Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"cba8f061-5c7f-4d25-be7c-69f58b879ecd","remote_addr":"[fd00:1122:3344:101::a]:34038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.822546388Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63524"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.824244331Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"047ab071-4749-4a4d-bcc6-fa676ba6bce1","remote_addr":"[fd00:1122:3344:101::b]:63524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.838913408Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"dc87ad06-a91c-4a84-92a9-b6da441cac18","remote_addr":"[fd00:1122:3344:101::b]:63524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":146,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.899515624Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62760"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.899705068Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"99ef14e6-ad05-4fe9-9749-12a4eab8b8f3","remote_addr":"[fd00:1122:3344:101::a]:62760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":165,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.922052386Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44026"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.922162958Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:33.964433096Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.032988148Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7d8063a4-3de0-4ef4-b56f-2fe4d4d9ab2e","remote_addr":"[fd00:1122:3344:101::a]:44026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110864,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.033239213Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33333"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.033284704Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56174"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.033358845Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35526"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.033447817Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"57b81636-b06f-448d-9942-ddb215dfd709","remote_addr":"[fd00:1122:3344:101::c]:33333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.033658421Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fc90ecaa-25e9-4eba-b68c-22d27ec46f63","remote_addr":"[fd00:1122:3344:101::a]:44026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":264,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.033729532Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"9a5fc231-7e02-4431-8017-b1ac00938c63","remote_addr":"[fd00:1122:3344:101::b]:56174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":325,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.03461596Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.043745149Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d2a96b78-d619-435c-b1f5-f1813f344a59","remote_addr":"[fd00:1122:3344:101::c]:33333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":307,"response_code":"204"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.080914816Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.098534621Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56504"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.098915719Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0d9f7765-f39c-4888-a447-6ffa200af4a8","remote_addr":"[fd00:1122:3344:101::c]:56504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":273,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.114264469Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57369"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.145821427Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.145879678Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"194effb3-a49e-4fee-afe7-30bd701ebccd","remote_addr":"[fd00:1122:3344:101::b]:35526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112307,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.150070731Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bbd1b2b8-82f3-4e82-9166-823321a3fbed","remote_addr":"[fd00:1122:3344:101::b]:35526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":657,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.185024535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.246318175Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3e3775a5-b9a2-4889-a32b-278bedf20fe1","remote_addr":"[fd00:1122:3344:101::c]:57369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":131936,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:34.246725513Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2e4924e7-15db-4d84-aa6b-c81c2d4d6146","remote_addr":"[fd00:1122:3344:101::c]:57369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.390611262Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"bada2397-aff2-4c7a-bd02-958f17fcdda8","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6063,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.609377556Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34612"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.609531569Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f3250ae0-cea3-4d86-a0a0-72883794d780","remote_addr":"[fd00:1122:3344:101::a]:34612","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"}
{"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.610322434Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:778","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(7604531) }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.610343625Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"PUT","req_id":"54d34ca8-0b93-4ed6-9b5e-9894ebf12682","remote_addr":"[fd00:1122:3344:101::a]:34612","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":630,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.71886918Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62095"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.720330159Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:570","metadata":"InstanceMetadata { silo_id: 50153162-da63-464a-9827-7da368e797d8, project_id: 8a74de03-5df8-4ae4-87eb-7eb972345fae }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-29T15:24:42.140203Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(88d1c68d-bdb0-4b5b-82b9-a01a804ae13c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-29T15:24:42.216581335Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(7604531))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-5788b950fad\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 2624347c-7537-49af-8ab7-cdee418cb0cc, read_only_parent: Some(Volume { block_size: 512, id: 3834d461-5285-4dc4-8d41-e0ac17dcc16f, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 387724df-00cf-41bf-93e8-43ee618611a9, key: Some(\"4ZJaAAbgLboYhE+wnCeyBEZfLToOAYoqqBAr0xTDJ68=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2624347c-7537-49af-8ab7-cdee418cb0cc, key: Some(\"wJiRsXlDbVgqH0sx31rnOruxThpcUI/txHGg4G/M5no=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], cloud_init_bytes: Some(..) }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.72039601Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:603","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.72042547Z","hostname":"sock","pid":1562,"propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:991","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(7604531))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 574b66c1-2c73-470c-8c17-6aeeff1266b9, kind: Instance { id: dea8d9be-c41c-461e-a0e0-e5b03267d5da }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 121, 135])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7604531), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-5788b950fad\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 2624347c-7537-49af-8ab7-cdee418cb0cc, read_only_parent: Some(Volume { block_size: 512, id: 3834d461-5285-4dc4-8d41-e0ac17dcc16f, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 387724df-00cf-41bf-93e8-43ee618611a9, key: Some(\"4ZJaAAbgLboYhE+wnCeyBEZfLToOAYoqqBAr0xTDJ68=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2624347c-7537-49af-8ab7-cdee418cb0cc, key: Some(\"wJiRsXlDbVgqH0sx31rnOruxThpcUI/txHGg4G/M5no=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-29T15:24:42.140203Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.721690815Z","hostname":"sock","pid":1562,"uri":"/vmms/88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","method":"PUT","req_id":"2ca21154-66cb-4521-96ba-e765b87a7089","remote_addr":"[fd00:1122:3344:101::a]:62095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2725,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.795195474Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62824"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.795369488Z","hostname":"sock","pid":1562,"uri":"/vmms/88d1c68d-bdb0-4b5b-82b9-a01a804ae13c/state","method":"PUT","req_id":"2fbbeede-1006-475b-a72e-107a6494c553","remote_addr":"[fd00:1122:3344:101::a]:62824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":125,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.796088902Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 245, 121, 135]), slot: 0, vni: Vni { inner: 7604531 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.796631443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.834740818Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.858060535Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.918788084Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51583"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:42.918950897Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"64d4e5fa-8040-4ae6-b919-5be6d50cdea7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51583","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"}
{"msg":"Profile for oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlInstance0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::1:0'/>\n        <propval type=\"astring\" name=\"listen_port\" value='12400'/>\n        <propval type=\"astring\" name=\"metric_addr\" value='dns'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:43.841549371Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:43.841761196Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 60.662009ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:44.058116712Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 94.693895ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:44.143019664Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 164.720714ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:44.254101199Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 537.568705ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:44.435990311Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 506.380155ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:44.989885456Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 1.230023474s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:45.508730185Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 1.478103269s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:46.750106202Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 1.012909382s","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:48.239751899Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.280858245Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:1449"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 61.498647ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:49.288751189Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 132.282119ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:49.359819861Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\") failed: Property not found. retry in 235.059755ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:24:49.501307641Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.749042492Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:1458"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.787948544Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:1482"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.933693267Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-08-29T15:24:49.936481141Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.936517722Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.936561573Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-29T15:24:49.936481141Z }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.936585194Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-29T15:24:49.936481141Z }, migration_in: None, migration_out: None }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-08-29T15:24:49.954060186Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.954099237Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.954119057Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-29T15:24:49.954060186Z }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:24:49.954129417Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-29T15:24:49.954060186Z }, migration_in: None, migration_out: None }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:03.907223193Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54979"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:03.907478538Z","hostname":"sock","pid":1562,"uri":"/vmms/88d1c68d-bdb0-4b5b-82b9-a01a804ae13c/state","method":"GET","req_id":"163aa6f7-ba27-4250-b4da-a067ab15739b","remote_addr":"[fd00:1122:3344:101::a]:54979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":223,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.020563233Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56498"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.020766357Z","hostname":"sock","pid":1562,"uri":"/vmms/88d1c68d-bdb0-4b5b-82b9-a01a804ae13c/state","method":"GET","req_id":"539ebcb0-d5e2-4ef3-804f-eccde164dc36","remote_addr":"[fd00:1122:3344:101::b]:56498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.039977033Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61869"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.040071935Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"4fdea449-ae7e-4b4a-b6a4-5b990aea894a","remote_addr":"[fd00:1122:3344:101::b]:61869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.074307745Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50634"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.075013109Z","hostname":"sock","pid":1562,"uri":"/vmms/88d1c68d-bdb0-4b5b-82b9-a01a804ae13c/state","method":"GET","req_id":"98634cd8-c191-4702-8fae-290e16b68085","remote_addr":"[fd00:1122:3344:101::c]:50634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":664,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.095888978Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37411"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.09598731Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c3b07144-0e9d-4d89-b6dd-90189aaba97d","remote_addr":"[fd00:1122:3344:101::c]:37411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.125954307Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51293"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.126073139Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"162a6af8-8c53-4871-bdfd-e31881093246","remote_addr":"[fd00:1122:3344:101::a]:51293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":81,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.179388413Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61572"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.179625547Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"1360f907-3938-4ba1-b8f3-dec89c380fc9","remote_addr":"[fd00:1122:3344:101::b]:61572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":186,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.183171387Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44916"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.1833247Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b0a20429-9bc9-4462-bf6c-42c7f0cd3509","remote_addr":"[fd00:1122:3344:101::c]:44916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.230226068Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1577d1b6-0cd1-41e1-a2a2-db9ed5cd26ba","remote_addr":"[fd00:1122:3344:101::c]:37411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":263,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.232624345Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53823"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.233449271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.27423039Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.332342688Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7a77288f-8cdb-48dc-b611-e03f55ab99b0","remote_addr":"[fd00:1122:3344:101::c]:53823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99660,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.332804387Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47357"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.332843187Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37968"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.332854858Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6d55019e-f674-4ff6-a546-91ed82fc8811","remote_addr":"[fd00:1122:3344:101::a]:51293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":318,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.332936329Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54960"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.338195252Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4c81dc13-79e9-4332-9ca4-22f8da55e292","remote_addr":"[fd00:1122:3344:101::b]:61869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5289,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.338382026Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"dbee400a-b9f5-4472-b087-a70f21b07434","remote_addr":"[fd00:1122:3344:101::c]:53823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5512,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.338628241Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.338717262Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"7445e8d0-c193-4051-bcce-53f9a462f74f","remote_addr":"[fd00:1122:3344:101::a]:54960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":502,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.377858479Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.435199272Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.435249613Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"203f3f29-fec0-43fe-aef4-2a4df47d39a0","remote_addr":"[fd00:1122:3344:101::a]:47357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97015,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.441018946Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6e5867df-cdcf-419b-abae-b96d92306569","remote_addr":"[fd00:1122:3344:101::a]:47357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":616,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.473711136Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4062"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.529033329Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d0758f5c-bf04-46ba-9026-3f62dc6d0d3d","remote_addr":"[fd00:1122:3344:101::b]:37968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":190790,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:04.529430887Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"30f82f78-e5fe-4e6c-86c3-21966c485fad","remote_addr":"[fd00:1122:3344:101::b]:37968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.838695662Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52350"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.839443617Z","hostname":"sock","pid":1562,"uri":"/vmms/88d1c68d-bdb0-4b5b-82b9-a01a804ae13c/state","method":"PUT","req_id":"b02315f6-a381-4037-a5ab-c29dc62a4a34","remote_addr":"[fd00:1122:3344:101::a]:52350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":704,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-08-29T15:25:07.839582579Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.8396048Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.83962941Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-29T15:25:07.839582579Z }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.83964277Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-29T15:25:07.839582579Z }, migration_in: None, migration_out: None }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-08-29T15:25:07.879362368Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.879395059Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.879428349Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-08-29T15:25:07.879362368Z }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.87945297Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:630","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.879926319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:25:07.880050932Z","hostname":"sock","pid":1562,"component":"kstat-sampler-worker","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":6,"id":"5755691835433126538"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.880219665Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:07.880280066Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/5a85a3ef-48f8-4333-ae7e-46dfaa7fa523/debug/bundle/zone\", \"/pool/int/6dd67a4e-189f-47cd-a6e2-0eff97293dab/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/05effe78-0c08-40b4-916b-cde344744ec3/crypt/debug\", \"/pool/ext/866d818f-82bb-444c-992a-1351dec91efa/crypt/debug\", \"/pool/ext/4fbbfff0-e175-4404-8cd9-acfc358b7140/crypt/debug\", \"/pool/ext/6eaf0007-0aa9-44b6-8bac-b01569f4c6a3/crypt/debug\", \"/pool/ext/775439b8-d6ea-4b72-9708-a30c4c7c6d6f/crypt/debug\", \"/pool/ext/e4f77dae-534d-4b73-b38a-890d525d3747/crypt/debug\", \"/pool/ext/1b55399c-9dbb-4d32-bd51-426a7996ea24/crypt/debug\", \"/pool/ext/aad99d77-5c0c-4ae7-8cc1-ced403d6b98a/crypt/debug\", \"/pool/ext/f4912882-4c2a-4c0d-b986-5cbff960587e/crypt/debug\"] }","zone_name":"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c"}
{"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:25:07.880549481Z","hostname":"sock","pid":1562,"component":"kstat-sampler-worker","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":6,"id":"6325694712850844852"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:08.507008918Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1137","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c\", bundle_id: 30d1d231-0287-4f3d-bbfc-f4bbf08de0e7 }, time_created: 2024-08-29T15:25:07.881197054Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","v":0,"name":"SledAgent","level":40,"time":"2024-08-29T15:25:08.507072299Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:826"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.273614308Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.291963127Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(7604531), kind: System }"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.292012578Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(7604531), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.292402326Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-29T15:25:10.292369135Z }, migration_in: None, migration_out: None }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.344470095Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-29T15:25:10.292369135Z }, migration_in: None, migration_out: None }","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.349370651Z","hostname":"sock","pid":1562,"instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","propolis_id":"88d1c68d-bdb0-4b5b-82b9-a01a804ae13c","instance_id":"dea8d9be-c41c-461e-a0e0-e5b03267d5da","component":"InstanceManager","file":"sled-agent/src/instance.rs:763"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.780026184Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40160"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.780116536Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"26445dd6-3ff1-417b-949d-1967cff8ba7d","remote_addr":"[fd00:1122:3344:101::a]:40160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.789794665Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"426cb7b3-e7a6-457f-9e3a-71a955f6e0fe","remote_addr":"[fd00:1122:3344:101::a]:40160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.99178701Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58390"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:10.991957383Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"04bb7609-e751-483c-aa2d-2ba84123656d","remote_addr":"[fd00:1122:3344:101::a]:58390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:11.312168343Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65131"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:11.312331656Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"fbba3dad-17d4-41d0-af24-21aaecc8957f","remote_addr":"[fd00:1122:3344:101::a]:65131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:12.505259685Z","hostname":"sock","pid":1562,"uri":"/b9c40332-a231-42db-adf0-b267d663a311","method":"GET","req_id":"32d0b532-5747-41b4-8094-a759ce9bed42","remote_addr":"[fd00:1122:3344:101::d]:51670","local_addr":"[fd00:1122:3344:101::1]:34899","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"b9c40332-a231-42db-adf0-b267d663a311","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6084,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:12.647870017Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59296"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:12.64799396Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"45e95119-4cd1-4998-bd64-7b3295a94e81","remote_addr":"[fd00:1122:3344:101::a]:59296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":80,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:12.65567189Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"afef4074-c109-4618-9b93-de298ac444c5","remote_addr":"[fd00:1122:3344:101::a]:59296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:12.953344318Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37872"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-29T15:25:12.953409869Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8535ae4a-a52c-44ba-beb4-8df52ad1d8fe","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37872","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"}