[ Sep 10 21:21:28 Enabled. ]
[ Sep 10 21:21:28 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Sep 10 21:21:28 Method "start" exited with status 0. ]
[ Sep 10 21:21:28 Rereading configuration. ]
[ Sep 10 21:21:28 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.493462291Z","hostname":"sock","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.493630374Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.544661624Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.544707925Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.658871737Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.667422526Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:28.675013498Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.272280059Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.414525881Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.414554342Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.414602093Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.414659084Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.414692844Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.414714575Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.417655216Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:29.417662006Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.417685076Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.417691257Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.417719507Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.417771358Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.41787451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.41789566Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.417909431Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.428034937Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.428056897Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.526433234Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936227174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936274975Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936282545Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936294756Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936302856Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936308686Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.936327226Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.944637201Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:29.944660931Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.034052881Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431094709Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431138559Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43117896Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43118663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43119292Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43119671Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.4312013Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431212111Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431225861Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431233931Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431238781Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431254031Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431257921Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431261731Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431280022Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431287852Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431292462Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431297532Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431302232Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431341743Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431361553Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431365783Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431377414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431381803Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431403114Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431410474Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431432654Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431438344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431443484Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431446705Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431450575Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431457415Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431483265Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431489215Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431493196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431501205Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431506216Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431509836Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431530786Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431542106Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431545906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431549117Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431563217Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431577997Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431583507Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431604727Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431622648Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431627358Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431630808Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431635668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431638928Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431642818Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431671908Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431678199Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431681439Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431692079Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431695629Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431701589Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43172836Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43173251Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43173579Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43173914Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43175252Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43176082Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43176876Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43177573Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.43177961Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431804251Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.431808671Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.442291474Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:730"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.442321724Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999946629s","next_cleanup":"Instant { tv_sec: 2842, tv_nsec: 871209758 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.442342405Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3842"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.442401736Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.444632645Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.444694326Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445036362Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445102323Z","hostname":"sock","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV2 {\n        rack_subnet: Ipv6Net {\n            addr: fd00:1122:3344:100::,\n            width: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV2 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Net {\n                                addr: 0.0.0.0,\n                                width: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                        vlan_id: None,\n                        local_pref: None,\n                    },\n                ],\n                addresses: [\n                    UplinkAddressConfig {\n                        address: V4(\n                            Ipv4Net {\n                                addr: 10.151.2.174,\n                                width: 24,\n                            },\n                        ),\n                        vlan_id: None,\n                    },\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n                lldp: None,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445334817Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445343487Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445418038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445479949Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.44550209Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.44551174Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445541321Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445553521Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445562571Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445566481Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445569751Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445574821Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445602732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.445631012Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446000569Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage: Plan {\n    rack_id: ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75,\n    sleds: {\n        [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n            generation: 0,\n            schema_version: 1,\n            body: StartSledAgentRequestBody {\n                id: 403254df-bc94-46d0-9b33-45b17d93603a,\n                rack_id: ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75,\n                use_trust_quorum: false,\n                is_lrtq_learner: false,\n                subnet: Ipv6Subnet {\n                    net: Ipv6Net {\n                        addr: fd00:1122:3344:101::,\n                        width: 64,\n                    },\n                },\n            },\n        },\n    },\n    config: RackInitializeRequest {\n        trust_quorum_peers: None,\n        bootstrap_discovery: OnlyOurs,\n        ntp_servers: [\n            \"0.pool.ntp.org\",\n        ],\n        dns_servers: [\n            1.1.1.1,\n            9.9.9.9,\n        ],\n        internal_services_ip_pool_ranges: [\n            V4(\n                Ipv4Range {\n                    first: 10.151.2.164,\n                    last: 10.151.2.173,\n                },\n            ),\n        ],\n        external_dns_ips: [\n            10.151.2.164,\n            10.151.2.165,\n        ],\n        external_dns_zone_name: \"oxide.test\",\n        external_certificates: \"<redacted>\",\n        recovery_silo: RecoverySiloConfig {\n            silo_name: Name(\n                \"recovery\",\n            ),\n            user_name: UserId(\n                \"recovery\",\n            ),\n            user_password_hash: NewPasswordHash(\n                PasswordHashString {\n                    string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                    encoding: B64,\n                },\n            ),\n        },\n        rack_network_config: RackNetworkConfigV2 {\n            rack_subnet: Ipv6Net {\n                addr: fd00:1122:3344:100::,\n                width: 56,\n            },\n            infra_ip_first: 10.151.2.174,\n            infra_ip_last: 10.151.2.174,\n            ports: [\n                PortConfigV2 {\n                    routes: [\n                        RouteConfig {\n                            destination: V4(\n                                Ipv4Net {\n                                    addr: 0.0.0.0,\n                                    width: 0,\n                                },\n                            ),\n                            nexthop: 10.151.2.100,\n                            vlan_id: None,\n                            local_pref: None,\n                        },\n                    ],\n                    addresses: [\n                        UplinkAddressConfig {\n                            address: V4(\n                                Ipv4Net {\n                                    addr: 10.151.2.174,\n                                    width: 24,\n                                },\n                            ),\n                            vlan_id: None,\n                        },\n                    ],\n                    switch: Switch0,\n                    port: \"qsfp0\",\n                    uplink_port_speed: Speed40G,\n                    uplink_port_fec: None,\n                    bgp_peers: [],\n                    autoneg: false,\n                    lldp: None,\n                },\n            ],\n            bgp: [],\n            bfd: [],\n        },\n        allowed_source_ips: Any,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446300474Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446331084Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446335524Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446350765Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446493017Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.44662226Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.44665187Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:221","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 403254df-bc94-46d0-9b33-45b17d93603a, rack_id: ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446787832Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:49508","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 403254df-bc94-46d0-9b33-45b17d93603a, rack_id: ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446884554Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446890764Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446918445Z","hostname":"sock","pid":1562,"file":"sled-agent/src/server.rs:44"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446927225Z","hostname":"sock","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446931715Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446935215Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446941765Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446959835Z","hostname":"sock","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446971686Z","hostname":"sock","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446977406Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446981326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.446984466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.447016136Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.447020196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.449292836Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4139"}
{"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.449343997Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4521"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.480083053Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.500605011Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.517912663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.524394426Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:390"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.524421687Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.524439297Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:402"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.524494718Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 77.398674ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.527383029Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7545601c-19f5-4ce3-8a14-1bcf0e9970af\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7545601c-19f5-4ce3-8a14-1bcf0e9970af\" }"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_d8f3d243-e151-404c-ad0f-38c8ce21760b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.536932545Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_d8f3d243-e151-404c-ad0f-38c8ce21760b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.558537542Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_d8f3d243-e151-404c-ad0f-38c8ce21760b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.558571133Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.563402897Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:417"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.563411017Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:130"}
{"msg":"Ensuring dataset oxi_d8f3d243-e151-404c-ad0f-38c8ce21760b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.563415727Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:140"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.605818637Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 60.452351ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.606369427Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a699e1ed-ce8f-4135-b6b1-c5558c2dc58b\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a699e1ed-ce8f-4135-b6b1-c5558c2dc58b\" }"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.661280345Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:175"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.667565544Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 212.650828ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.668162195Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ad0262e3-224a-468b-804a-0c56c2f5426c\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ad0262e3-224a-468b-804a-0c56c2f5426c\" }"}
{"msg":"Mounting oxi_d8f3d243-e151-404c-ad0f-38c8ce21760b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.879706876Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:183"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.882188749Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 243.070767ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.883037204Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5b7a52aa-12e4-4012-936b-8344744a89f5\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5b7a52aa-12e4-4012-936b-8344744a89f5\" }"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.893112Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:191"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.893200552Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:191"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.893230282Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:191"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:30.893265993Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:197"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.127859666Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 200.028247ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.128994185Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25fb2208-a87e-410f-81e3-cf065f9099b3\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25fb2208-a87e-410f-81e3-cf065f9099b3\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.330520542Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 228.996474ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.331170773Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb70445d-b319-4726-9cdf-51dec08b89f5\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb70445d-b319-4726-9cdf-51dec08b89f5\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.561267828Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 280.191007ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.561935709Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"08e61c2c-38db-4f9f-9edc-884da24ef3b0\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"08e61c2c-38db-4f9f-9edc-884da24ef3b0\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.623564495Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.62901775Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.629064381Z","hostname":"sock","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.629131212Z","hostname":"sock","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:48774","producer_id":"403254df-bc94-46d0-9b33-45b17d93603a"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:31.630306433Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"155.740827ms"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.631165148Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.631171648Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.631254499Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:914","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.631259999Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:31.78833705Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"340.113235ms"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:31.843192927Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:32.12997568Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"871.482993ms"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:33.003124395Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.250348566s"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.027676591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2615"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.027698522Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2833"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.027703412Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2485"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.027719012Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2532"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.027723042Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2914"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.027726742Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.029624665Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2758"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.030920848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2526"}
{"msg":"Profile for oxz_switch:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgs\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"id\" value='3817645b-bf82-4294-ab73-827679215664'/>\n        <propval type=\"astring\" name=\"rack_id\" value='ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/wicketd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12226'/>\n        <propval type=\"astring\" name=\"artifact-address\" value='[fdb0:18c0:4d0d:a02a::2]:12227'/>\n        <propval type=\"astring\" name=\"baseboard-file\" value='/opt/oxide/baseboard.json'/>\n        <propval type=\"astring\" name=\"mgs-address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"nexus-proxy-address\" value='[::]:12229'/>\n        <propval type=\"astring\" name=\"rack-subnet\" value='fd00:1122:3344::'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/switch_zone_setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"gz_local_link_addr\" value='fe80::8:20ff:fe16:620'/>\n        <property type=\"astring\" name=\"link_local_links\">\n          <astring_list>\n            <value_node value=\"tfportrear0_0\" />\n            <value_node value=\"oxBootstrap0\" />\n          </astring_list>\n        </property>\n        <propval type=\"astring\" name=\"bootstrap_addr\" value='fdb0:18c0:4d0d:a02a::2'/>\n        <propval type=\"astring\" name=\"bootstrap_vnic\" value='oxBootstrap0'/>\n        <propval type=\"astring\" name=\"baseboard_info\" value='{\"type\":\"pc\",\"identifier\":\"sock\",\"model\":\"i86pc\"}'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/dendrite\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_id\" value='403254df-bc94-46d0-9b33-45b17d93603a'/>\n        <propval type=\"astring\" name=\"rack_id\" value='ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12224'/>\n        <propval type=\"astring\" name=\"front_ports\" value='1'/>\n        <propval type=\"astring\" name=\"rear_ports\" value='1'/>\n        <propval type=\"astring\" name=\"port_config\" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>\n        <propval type=\"astring\" name=\"mgmt\" value='uds'/>\n        <propval type=\"astring\" name=\"uds_path\" value='/opt/softnpu/stuff'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/tfport\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"host\" value='[::1]'/>\n        <propval type=\"astring\" name=\"port\" value='12224'/>\n        <propval type=\"astring\" name=\"flags\" value='--sync-only'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/lldpd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"board_rev\" value='softnpu_front_1_rear_1'/>\n        <propval type=\"astring\" name=\"scrimlet_id\" value='sock'/>\n        <propval type=\"astring\" name=\"scrimlet_model\" value='i86pc'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12230'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/pumpkind\">\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_uuid\" value='403254df-bc94-46d0-9b33-45b17d93603a'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mg-ddm\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"mode\" value='transit'/>\n        <propval type=\"astring\" name=\"dendrite\" value='true'/>\n        <propval type=\"astring\" name=\"sled_uuid\" value='403254df-bc94-46d0-9b33-45b17d93603a'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75'/>\n        <propval type=\"astring\" name=\"interfaces\" value='tfportrear0_0/ll'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/uplink\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.03101528Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:34.031243884Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 70.916881ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:34.410047803Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 146.310977ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:34.603826274Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 220.443934ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:34.780764271Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 249.437544ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:35.122329591Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:35.255545975Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.360433768s"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.122157559s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:35.515746566Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 778.110736ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:36.802497167Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.487374154s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:37.66808322Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:38.618317159Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"9.92429519s"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.41144678s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:39.293547741Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.078883241Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.078930972Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.078937842Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.078948282Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.079158876Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.079321219Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.079357529Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.079368759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3842"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.07940558Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:49508","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.306741367Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:112","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.306823888Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:236","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.307049302Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.307119823Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.307176734Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.307196575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.307206025Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.307225385Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.31209498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4163","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.31210487Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4179"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.31211511Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.535752343Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.56481031Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39817"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.565089244Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.565232897Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"b08b3d06-4702-4e2a-8790-7f0a3a300b63","remote_addr":"[fd00:1122:3344:101::1]:39817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":286,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.595744889Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39297"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.595829871Z","hostname":"sock","pid":1562,"uri":"/sled-role","method":"GET","req_id":"1317fe0f-8e62-4389-bf51-6b0f74799ce6","remote_addr":"[fd00:1122:3344:101::1]:39297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.597311537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.597339547Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.597710934Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.597975518Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:1001"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 0b588f92-4852-4d93-afd5-f605d2829fd5, pool_id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 96226f68-ef50-40f9-8b33-cb0df72481aa, pool_id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 44248c05-c585-480e-af0b-37285b475545, pool_id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 8186d965-cbcd-43b6-a6ec-35f685f82392, pool_id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 5f0bf544-f986-4cb3-a51f-e8e7a6929122, pool_id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: ecd5ec5f-d997-4476-ab73-77999669d2fe, pool_id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 5572178f-0a59-4a8b-a4ec-5c81351ce1c7, pool_id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6bc8374b-c63f-4671-b42a-a3d4d03849dc, pool_id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 3f1155d0-d972-435c-a887-0bebd422e41c, pool_id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630059578Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.63015604Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45077"}
{"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630416894Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:858"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 0b588f92-4852-4d93-afd5-f605d2829fd5, pool_id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 96226f68-ef50-40f9-8b33-cb0df72481aa, pool_id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 44248c05-c585-480e-af0b-37285b475545, pool_id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 8186d965-cbcd-43b6-a6ec-35f685f82392, pool_id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 5f0bf544-f986-4cb3-a51f-e8e7a6929122, pool_id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: ecd5ec5f-d997-4476-ab73-77999669d2fe, pool_id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 5572178f-0a59-4a8b-a4ec-5c81351ce1c7, pool_id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6bc8374b-c63f-4671-b42a-a3d4d03849dc, pool_id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 3f1155d0-d972-435c-a887-0bebd422e41c, pool_id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630475965Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630481576Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630492376Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630497226Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630528596Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630534397Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630539506Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.630549487Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.640643093Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.640664423Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.739686941Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747166641Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747210032Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747219302Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747228093Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747237703Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747394505Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.747402996Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.782884585Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.790717771Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.965028733Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.97291983Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.97295078Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.972959771Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.972968421Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.972995821Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.982059979Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:41.98209622Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.072733441Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080226852Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080279533Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080294093Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080339244Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080354325Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080512477Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.080519497Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:21:42.084417685Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.115921375Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_1d09a0b5-64fe-49da-9f0c-2aeb05367248/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.124962173Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_1d09a0b5-64fe-49da-9f0c-2aeb05367248","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.301042405Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.309177077Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.309209558Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.309216488Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.309222198Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.309226058Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.319016139Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.319041249Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.412553691Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420220815Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420271426Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420278736Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420300246Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420307216Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420435008Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.420444229Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.453831811Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_b41c1bbc-10c8-46e3-9c3a-722fb6b38e83/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.462187187Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_b41c1bbc-10c8-46e3-9c3a-722fb6b38e83","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.63942967Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.647583942Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.647621443Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.647627273Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.647635703Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.647654033Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.657626047Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.657665438Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.751508255Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759093648Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759149578Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759157519Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759173779Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759184459Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759313352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.759320341Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.7953279Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_233394d8-3133-44e6-a8c8-baa69c3f73c6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.802828351Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_233394d8-3133-44e6-a8c8-baa69c3f73c6","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.9776043Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.985369925Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.985408866Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.985413936Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.985418256Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.985421966Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.995186556Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:42.995206377Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.092386042Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.100085297Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.100109427Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.100115597Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.100143608Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.100152058Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.10028954Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.100304061Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:21:43.103918084Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.13580152Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_a1f44de4-bbe9-4024-862c-8b1b1210d80a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.143057436Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_a1f44de4-bbe9-4024-862c-8b1b1210d80a","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.312067425Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.319598217Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.319635948Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.319660708Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.319666028Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.319670248Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.329238105Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.329273246Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.424624679Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432757671Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432788222Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432794032Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432802992Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432812162Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432985295Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.432992445Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.468871081Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_eb1402a5-e990-4914-9b22-a55b7b60e97b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.476944772Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_eb1402a5-e990-4914-9b22-a55b7b60e97b","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.652871042Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.661254678Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.661290379Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.661300739Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.661319649Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.661325479Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.670882076Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.670906767Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.767473842Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.775342419Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.775376389Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.77538264Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.77541598Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.77542905Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.775689325Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.775694825Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.812408576Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.820542608Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:43.99951263Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.006552653Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.006588793Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.006596363Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.006601294Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.006605503Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.01497855Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.01500494Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"e82c3049-23d1-4237-ad1f-14e37401e51a (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.117201773Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.124208595Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.124243716Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.124249596Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.124313667Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.124327748Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.12444132Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.12444887Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:21:44.127569454Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.158000365Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_e82c3049-23d1-4237-ad1f-14e37401e51a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.165115169Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_e82c3049-23d1-4237-ad1f-14e37401e51a","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.33536133Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.343171576Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.343207517Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.343216557Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.343224777Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.343252668Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.353103229Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.35312992Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"099b9246-cdfd-4461-86ab-695e2d382672 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.460712197Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:185","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470219443Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:229","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470254364Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470261674Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:235","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470292534Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470299854Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470433737Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.470443807Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:249"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.510026878Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_099b9246-cdfd-4461-86ab-695e2d382672/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.518455935Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:292"}
{"msg":"Looking for unencrypted datasets in oxp_099b9246-cdfd-4461-86ab-695e2d382672","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.691935001Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:378"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700145635Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700191846Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700285257Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700394739Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d8f3d243-e151-404c-ad0f-38c8ce21760b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 473e7fed-bfef-4f22-b3d1-ee8652774c28 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.70045291Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.70045946Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700574742Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700813287Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:862"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700839117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700843367Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:888","generation":"Generation(1)"}
{"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700847307Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"}
{"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700852127Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:898"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700860377Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700864957Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700902148Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:803","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700907038Z","hostname":"sock","pid":1562,"sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:909"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.700980619Z","hostname":"sock","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"874accec-c32d-44d0-968c-2e59843c4990","remote_addr":"[fd00:1122:3344:101::1]:45077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3070758,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.732449699Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.732602661Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38132"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.732973958Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.732980048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.784534247Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.784564378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.834289365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.834326766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.881727333Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.881771694Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.881806574Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.881855435Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.92509775Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.92513449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.925138971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.976199381Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.979809144Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.983922366Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:44.99503938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:45.000305402Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:45.004471354Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:21:45.129920783Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Profile for oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService1'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:2::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:2::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:2::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:2::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:46.514078354Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:46.514168116Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"}
{"msg":"Booting oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:46.514341409Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService2'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:3::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:3::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:3::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:3::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:46.997488759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService3'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:1::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:1::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:1::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:1::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:47.479468748Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:47.479526219Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:47.479536239Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 54.242379ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.487724042Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:47.487776963Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:47.491829394Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 105.112704ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.552469402Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 63.356084ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.618892061Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 30.890748ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.625239162Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 54.568953ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.663432048Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 102.154864ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.670149225Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 145.381223ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.891515818Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 256.161152ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.939100428Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 103.460409ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:47.960139035Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 193.650854ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.05324666Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 372.051743ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.075922635Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 682.397977ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.204533899Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 406.179396ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.25841466Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 797.187229ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.457794978Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 500.36574ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.674178294Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 1.354551692s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:48.897204026Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 852.07679ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:49.185409084Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 868.230148ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:49.264415322Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 858.030317ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:50.046659681Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 902.582578ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:50.14120003Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 831.276031ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:50.259698728Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.484793055Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4187"}
{"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.484845996Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4197","gateway":"fd00:1122:3344:101::1"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.491218667Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4276"}
{"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.612783848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4324"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.612830679Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4351"}
{"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.651069267Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4364"}
{"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.651105127Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4228"}
{"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.70573331Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4270"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.705787251Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4331","rack_subnet":"fd00:1122:3344::"}
{"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.72574874Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4342"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.7257834Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4384"}
{"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.813050983Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4416"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.813093864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4422"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 822.497081ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:50.914112566Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.923452199Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4463"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:50.92349395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"}
{"msg":"Failed to get switch ID from MGS (retrying in 59.170039ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:50.955995197Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Failed to get switch ID from MGS (retrying in 117.312255ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.016431841Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 724.048121ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.051212568Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 1.030318189s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.098396961Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Failed to get switch ID from MGS (retrying in 174.824022ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.134657194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Failed to get switch ID from MGS (retrying in 389.456626ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.310861999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.546342217Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:36276"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.547338985Z","hostname":"sock","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"57c87c06-8f5a-4cad-8685-51b6179e5bec","remote_addr":"[fd00:1122:3344:101::2]:36276","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":759,"response_code":"200"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.702219507Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.735992086Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.736100308Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.736538106Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"0"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.736552926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"wait for service Some(\"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea\") failed: Property not found. retry in 1.462425192s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.745447451Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.7825957Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 672.380097ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:51.786514408Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832788635Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832842687Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832853427Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832861787Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832867157Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832872467Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832877747Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832883067Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832888237Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832894467Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832899457Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832904508Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832909557Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832923398Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832929878Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832935078Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832940268Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832944298Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832948298Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832952258Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832956278Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832960249Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:51.832964269Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"wait for service Some(\"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616\") failed: Property not found. retry in 1.224946504s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:52.139021738Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1\") failed: Property not found. retry in 1.274456862s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:52.469152498Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:53.547419772Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"no record found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"8.083850959s"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.761223733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_internal_dns_b8ae7144-e4b7-44d2-8d80-34bfd3575616"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.761263383Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_internal_dns_94337c1c-5cb5-4050-bd90-d4c50dd633f1"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.761272754Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_internal_dns_48d3ebb5-3dc4-4b69-a505-4bb07f9482ea"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.761299194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.761741512Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.762083168Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"74e373ba-3ba0-4a2e-b377-0ae21c939264","remote_addr":"[fd00:1122:3344:101::1]:38132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9029349,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.763785618Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.810561604Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.86422913Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.912878419Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.912890069Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.945796123Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.946506286Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.946529226Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.975897768Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.97598694Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57196"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.976647551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.976653761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.998860569Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:53.998899109Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.017361222Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.017389112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.036452935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.036498915Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.036727539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.036896182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.04766006Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.079810301Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.080549294Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.080559794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.110960535Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 136, 116]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.110980135Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"boundary_ntp"}
{"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.111397822Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.111407612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.111756049Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.112166146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.136446349Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.155170996Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService4'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value='fd00:1122:3344:100::/56'/>\n        <propval type=\"boolean\" name=\"boundary\" value='true'/>\n        <propval type=\"astring\" name=\"boundary_pool\" value='boundary-ntp.control-plane.oxide.internal'/>\n        <propval type=\"astring\" name=\"server\" value='0.pool.ntp.org'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte0'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.3.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.3.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.770875809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:21:54.771065222Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 42.461746ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:54.892287338Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 87.551733ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:54.941539897Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 144.530862ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:55.057905357Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 462.554343ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:55.213693425Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 858.165017ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:55.685382205Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 569.275442ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:56.554407798Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 805.8102ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:57.133252208Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 1.113197861s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:57.947951083Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397\") failed: Property not found. retry in 1.171408112s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:21:59.070544499Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.261187804Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.261256665Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.261662752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.262072709Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"16e2ece7-0c1f-4196-87e0-41103baf5975","remote_addr":"[fd00:1122:3344:101::1]:57196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6286035,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.264715655Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.295442241Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.295615274Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42681"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:22:00.304729463Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.304897436Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"b002d4e4-4a55-4777-815e-05fc01fb3587","remote_addr":"[fd00:1122:3344:101::1]:42681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9155,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:00.307442031Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.461117332Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:00.461219034Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57855"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:22:04.469873957Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.469957739Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"d4a59399-4078-4f87-894e-cf2d2ebd14fe","remote_addr":"[fd00:1122:3344:101::1]:57855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008582,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:04.473758745Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.684079095Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.684245238Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51739"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:22:04.693472549Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.693628351Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"6a89d4f3-52a1-49db-8bcc-6629215ebb3b","remote_addr":"[fd00:1122:3344:101::1]:51739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9320,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:04.696036663Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.941599338Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.941789661Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46965"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_19b992ad-ed77-421f-ac97-09b7e8b99397': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:22:04.949563327Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:04.949669959Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"89782293-0b03-4798-93d8-bb38ddc0834d","remote_addr":"[fd00:1122:3344:101::1]:46965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7829,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:04.952174142Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:05.216472074Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:05.216615836Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49721"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.224751666Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ea159768-8ed3-40c2-96d7-d7e152025b90","remote_addr":"[fd00:1122:3344:101::1]:49721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1008088,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.226399165Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:06.226407135Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.437775183Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.437953646Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55986"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.44621758Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1c2c9942-a689-4e03-b9d0-d41d82c15ef9","remote_addr":"[fd00:1122:3344:101::1]:55986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8207,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.449235923Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:06.449256333Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.709484264Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.709665977Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47511"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.718113265Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"d7078628-bb46-4d88-9cd9-61f6207190b1","remote_addr":"[fd00:1122:3344:101::1]:47511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8392,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:06.721347751Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:06.721357851Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:07.13456334Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:07.134715303Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63052"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:07.142909076Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"6aba5345-cb27-46b5-91f5-d97043bb62e3","remote_addr":"[fd00:1122:3344:101::1]:63052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8163,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:07.145736115Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:07.145744755Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.243311196Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.243432298Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:32841"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.263196563Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"7e8e0665-6f9a-4944-a5b5-1c46fe1ae3e6","remote_addr":"[fd00:1122:3344:101::1]:32841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":19643,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1753455535, ip_addr: 104.131.155.175, stratum: 3, ref_time: 1726003327.7345097, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.264910293Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.264914493Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1299"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.294411507Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.29455788Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37333"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.294987077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.294993657Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.316306739Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.31634782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.33582624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.335861351Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.355143877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.355174057Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.415938738Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.415982429Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.467733112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.467779622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.5197721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.51981681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.570612996Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.570656657Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.624918084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.624965015Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.625195029Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:08.625385952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.050377839Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.05043297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.050440411Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.050444291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.050449471Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.138470576Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.142492546Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.146666479Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.150877822Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.155206808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.159508363Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.164270466Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.168386378Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.173563738Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:09.178550855Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService5'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::6]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='9dddae67-2068-4cf1-b0cb-d2382c7eda0c'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::6]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::6]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.601143055Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.601308755Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService8'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::5]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='71834d57-8f47-4ba6-a6ab-2f8adc6a35e3'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::5]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::5]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.663309752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.663523441Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService9'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::3'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::3]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='b7df98df-a73e-420a-ba60-6f5404c39291'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::3]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::3]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.670305259Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.67049191Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService7'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::4'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::4]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='61d20555-41e5-4532-a1c0-bfd0c4966c8d'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::4]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::4]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.704603084Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.704727314Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 48.797647ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:11.744393677Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService6'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::7'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::7]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='a2229031-29c9-471e-b908-4fd2e6787eb8'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::7]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::7]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.757245212Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:11.757391784Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 53.223026ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:11.803008849Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 140.701478ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:11.859969612Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 69.115463ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:11.865681758Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 139.726957ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:11.88905561Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 135.094267ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:11.998701329Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 120.07056ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.043757953Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 70.033517ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.076615717Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 293.594505ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.361299219Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 417.498431ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.415452403Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 74.568578ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.479183774Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 108.549243ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.599080587Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 142.615142ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.853488573Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 584.562393ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:12.89399935Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 994.363796ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.074608583Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 88.001366ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.189704102Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 198.746274ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.302798544Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 410.536747ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.448205152Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 116.969135ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.601794332Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 499.331127ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.758567597Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 1.11787027s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:13.947044078Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 343.989171ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:14.107199765Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 900.819801ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:14.21921336Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 1.160174521s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:14.231677634Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 992.853361ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:14.300233129Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 552.198462ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:14.463148902Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 877.917852ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.026731268Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 1.117607851s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.076931709Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 1.119835178s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.110399923Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 545.086717ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.304820313Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 785.178759ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.40525021Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.157956405s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.860350742Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.172290931s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:15.915990732Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 1.385825409s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:16.202381689Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 1.125863017s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:16.210931309Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 1.26148819s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:16.242309539Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 570.511237ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.032089825Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 837.745256ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.104702987Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 1.166522909s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.350291388Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 1.159883155s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.516595569Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 949.223035ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.60024866Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.460951157s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.611919973Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.01283126s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:17.953731573Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 794.546946ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:18.530273072Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 1.092229272s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:18.561774926Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 1.418147463s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:18.689018493Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 922.373603ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:18.979459462Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.132750635s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:19.087096225Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291\") failed: Property not found. retry in 1.15685165s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:19.336639826Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 666.644488ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:19.667254696Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 537.79287ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:19.914175672Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3\") failed: Property not found. retry in 1.328552067s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:20.122597233Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:20.150451004Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"348.267348ms"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.058536245s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:20.234115527Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c\") failed: Property not found. retry in 1.242428481s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:20.34712456Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.158797448s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:20.46488562Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 906.969633ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:21.306630159Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:21.311495657Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65192"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:21.311848867Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1a94c93f-e076-4043-8d21-7ecd2d12fcdf","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65192","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.369976114s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:21.636233004Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.359320797s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:22.234339305Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.106050706s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:23.020665832Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 512.793888ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:24.044041468Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 617.467881ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:24.149694262Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.363895609s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:24.569613652Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.4594773s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:24.780183411Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 781.876217ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:25.952931108Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 706.390571ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:26.257328882Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 660.218606ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:26.749057007Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 517.598823ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:26.975574793Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 867.29363ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:27.420246409Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 777.655558ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:27.502801863Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 957.782718ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:28.292243402Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.154674436s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:28.299369234Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8\") failed: Property not found. retry in 1.16769898s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:29.262683143Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d\") failed: Property not found. retry in 1.138326235s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:29.46679555Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.631678933Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_cockroachdb_9dddae67-2068-4cf1-b0cb-d2382c7eda0c"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.631734574Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_cockroachdb_a2229031-29c9-471e-b908-4fd2e6787eb8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.631743884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.631751712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_cockroachdb_71834d57-8f47-4ba6-a6ab-2f8adc6a35e3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.631824809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_cockroachdb_b7df98df-a73e-420a-ba60-6f5404c39291"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.631880628Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.632464101Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.641459824Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d48fb9cf-6d44-4b2f-a6ae-a7bd59a84f8b","remote_addr":"[fd00:1122:3344:101::1]:37333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":22437905,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.675698773Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63837"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.703665112Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"f8093f1e-001a-46fd-89b3-d7a8f94b0126","remote_addr":"[fd00:1122:3344:101::1]:63837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1  stdout: \n  stderr: ipadm: Could not get address: Object not found","latency_us":27737,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:30.703871741Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1028","retry_after":"142.509246ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8093f1e-001a-46fd-89b3-d7a8f94b0126\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8093f1e-001a-46fd-89b3-d7a8f94b0126\" }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.874400263Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"2bb00f90-e7ba-4913-be6d-3eee46e74bcc","remote_addr":"[fd00:1122:3344:101::1]:63837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_61d20555-41e5-4532-a1c0-bfd0c4966c8d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1  stdout: \n  stderr: ipadm: Could not get address: Object not found","latency_us":26755,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:30.874628438Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1028","retry_after":"79.124563ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2bb00f90-e7ba-4913-be6d-3eee46e74bcc\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2bb00f90-e7ba-4913-be6d-3eee46e74bcc\" }"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:30.982255756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3662"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:37.918939258Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3680"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.857143613Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3703"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.857261262Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"27aa5dd1-ba47-44d2-ab31-6d1052ea13f5","remote_addr":"[fd00:1122:3344:101::1]:63837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":12866778,"response_code":"204"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:43.864092576Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"534.276227ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.893213878Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.893351234Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52333"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.894059764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.894064883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.916048147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.91607767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.935318608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.935349603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.955315156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.955356369Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.974069353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.974096737Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.993167013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:43.993196883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.010669271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.010700472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.028824238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.028850489Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.047981808Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.048073462Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.152254805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.152356924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.266067142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.266103086Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.362056553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.362094204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.476708626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.47675012Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.549747359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.549790653Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.617858515Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.617893363Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.680517188Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.680554407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.75565434Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.755690998Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.820057026Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.820163414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.911876513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:44.911912288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.000828609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.000863747Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.082292326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.08233324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.082569606Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.082714603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204210135Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204266715Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204274224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204280225Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204302415Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204308697Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.20431329Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204319337Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204325394Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204332884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204340365Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204346395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204352798Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204361301Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204368735Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204374585Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204381635Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204387627Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204395089Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.204479006Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ae129fad-37e5-49e5-9223-6b38a2a0f46e","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized)]","latency_us":1307406,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.204645265Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"51.289795ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ae129fad-37e5-49e5-9223-6b38a2a0f46e\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ae129fad-37e5-49e5-9223-6b38a2a0f46e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.257192369Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.257610181Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.257618291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.276909431Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.276938868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.294918566Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.294950339Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.314378781Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.314406457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.331868468Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.331894513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.350904826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.350939393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.368847271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.368878837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.386926101Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.386952137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.405538223Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.405566929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.424230493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.424259922Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.443186028Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.443210123Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.461591728Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.461615372Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.482587346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.482619381Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.502746456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.502773345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.521684138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.521709536Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.539741035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.539768588Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.558418405Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.558445519Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.577493674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.577519184Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.595916218Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.595942225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.614805451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.614830923Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.632776241Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.63280402Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.63299833Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.633134288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725709677Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.72575267Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725759224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725766162Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725770268Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725775443Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725780843Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725786178Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.7258082Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725812607Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725817632Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725823228Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725828675Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725832247Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725835725Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725841097Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725844594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725848307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.725853416Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.726004263Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"91f142a5-047d-4fb7-ae0c-e17c06485541","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized)]","latency_us":467331,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:45.726162Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"108.038108ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91f142a5-047d-4fb7-ae0c-e17c06485541\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91f142a5-047d-4fb7-ae0c-e17c06485541\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.83590068Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.836684697Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.836692225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.858141575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.858181898Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.877459572Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.87749364Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.895226307Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.895259813Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.914739447Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.914776854Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.932192106Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.932222575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.949315298Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.949353435Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.967049652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.967086122Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.986056627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:45.986095469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.004617739Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.004653646Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.024440878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.024477301Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.042894456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.04293297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.061824271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.061858902Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.08074329Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.080779186Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.099862434Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.099902148Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.119054809Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.119102745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.138087359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.138119394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.156284222Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.156322482Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.17520525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.17523885Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.192948224Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.192984525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.212488486Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.212537369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.212748216Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.212947115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305733014Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305758984Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305762743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305766774Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305771434Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305775521Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305778643Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305781578Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305785357Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305789096Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.3057928Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.30579595Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305809291Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305813969Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305817785Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305822519Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305826307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305830113Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.305833938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.305939633Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ecbb2127-0535-43e8-8150-cbe270fbfb5b","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized)]","latency_us":468705,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.306084564Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"206.636453ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ecbb2127-0535-43e8-8150-cbe270fbfb5b\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ecbb2127-0535-43e8-8150-cbe270fbfb5b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.514081177Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.515099265Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.515105855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.537445233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.537475636Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.556923561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.556953636Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.576456697Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.576486023Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.597588142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.59761894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.616812169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.616854995Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.637923911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.637961326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.657140203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.657185823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.676655171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.676688809Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.694903887Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.694938876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.714516763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.714543547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.733076457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.733114511Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.751571531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.751599019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.770804957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.770842747Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.790514458Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.790546118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.809054756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.809085817Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.828782885Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.828829048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.847766624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.84781005Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.866494534Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.866528238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.884442878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.884478091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.902800977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.902844365Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.903068769Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:46.903209416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:46.999935194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000043174Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000048339Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000052652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.00005675Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000060837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000064653Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000068412Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000071309Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000075059Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000078772Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000082437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.00008539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000089215Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000092112Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000095834Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000098769Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000113994Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000119929Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.000245256Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4365e1d8-213b-47d9-a0cc-77b267ce7138","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized)]","latency_us":484702,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.000402262Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"225.37562ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4365e1d8-213b-47d9-a0cc-77b267ce7138\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4365e1d8-213b-47d9-a0cc-77b267ce7138\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.227289862Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.227588554Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.227661294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.251634889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.251732786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.271494081Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.271519019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.290647806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.290675051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.308013432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.308042561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.328482705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.328511009Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.347742563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.347771119Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.368146944Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.368176889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.386878248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.386907292Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.405783817Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.405815008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.425267941Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.425293582Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.444865046Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.444892544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.464755178Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.464783022Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.484234114Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.484272036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.504101821Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.504132891Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.525086539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.525115779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.54394002Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.543968352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.563460951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.563488823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.582095592Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.582122668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.601288836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.601315153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.620202919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.620237719Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.620434326Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.620553841Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712675421Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712725944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712732281Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712739209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712744591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712751651Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.71276727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712772717Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712778117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712784698Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712789817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712796117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712801114Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712806271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712812889Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712818055Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.71283463Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712849564Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.712857928Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.713060382Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"698249ea-edd9-44c4-bd14-09f0f9185eb9","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized)]","latency_us":484287,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:47.7132154Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"105.483665ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"698249ea-edd9-44c4-bd14-09f0f9185eb9\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"698249ea-edd9-44c4-bd14-09f0f9185eb9\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.819674499Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.819972151Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.819979961Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.842139699Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.842175418Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.861441195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.861480581Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.880687053Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.880725407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.89876479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.898801082Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.917770494Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.917820509Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.936844964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.936888409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.955740058Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.955795531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.974682026Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.974721101Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.994854272Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:47.994893253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.015325488Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.015366654Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.035376772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.035414431Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.054064498Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.054101727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.073518313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.073552767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.092092549Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.092130714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.110231942Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.110268037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.129360879Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.129396158Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.147548409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.147594319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.166867627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.166910181Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.186178384Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.186301218Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.205032035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.205066789Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.205379089Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.20553272Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300763324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300802878Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300807734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300810762Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300813781Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300817578Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.30082055Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300825387Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300831959Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300835747Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300846201Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300849257Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300853944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300857619Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300861416Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300866207Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300869994Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300873679Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.300877588Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.300966109Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ba0548f6-ed72-4ccc-9a9d-9abbfa7736a3","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized)]","latency_us":479868,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.301112615Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"189.087371ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba0548f6-ed72-4ccc-9a9d-9abbfa7736a3\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ba0548f6-ed72-4ccc-9a9d-9abbfa7736a3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.492109736Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.492386594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.492393559Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.515231743Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.515262156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.536122132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.536213474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.556415954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.556450295Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.574847235Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.57487208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.592873941Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.592899226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.611679628Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.611709365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.6297008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.629733904Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.649005731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.649034213Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.668062617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.668095168Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.687324007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.687349825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.707536063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.707574013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.727322989Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.727348564Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.745403429Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.745433899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.764595966Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.764629379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.783901203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.78393185Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.803374518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.803400853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.822576293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.822607485Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.840471104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.840499285Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.858851769Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.858876219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.877481442Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.877518426Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.877736342Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.877855632Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.97165151Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971689123Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971696642Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971701817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971706907Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971712176Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971717407Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971722442Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971727936Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971733308Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971737921Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.97174394Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971749396Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971763018Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971777118Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971782068Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971790722Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971796909Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.971801728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:48.971921599Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c9a95464-2536-400d-92d9-f0c3475df1e7","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized)]","latency_us":478442,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:48.972073242Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"193.781922ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9a95464-2536-400d-92d9-f0c3475df1e7\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c9a95464-2536-400d-92d9-f0c3475df1e7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.167083412Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.167369635Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.167375514Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.189021226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.189064108Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.208796769Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.208838545Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.226540698Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.226588868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.245901091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.245942032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.264006417Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.26404524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.28225727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.282299757Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.301553659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.30159611Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.319478348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.319515576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.338484885Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.338517333Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.356471028Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.356507076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.375879478Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.375919239Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.396260754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.396295995Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.415202591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.415240214Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.434082661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.434118971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.452321209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.452351791Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.471513793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.471556553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.490558674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.490610238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.509675065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.509717975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.528913184Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.528947721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.5481077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.548144507Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.548375577Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.548500622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641668565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641700609Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641705794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.64172522Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641729495Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641734248Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.64173887Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641741832Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641746351Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.64175011Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641754085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641758548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641762157Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641765888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641770435Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641773998Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641778452Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641782061Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641784902Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:49.641852956Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"26ff9cfa-7fbf-4676-8463-083806749ac4","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized)]","latency_us":473413,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:49.641995731Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"437.202724ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26ff9cfa-7fbf-4676-8463-083806749ac4\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"26ff9cfa-7fbf-4676-8463-083806749ac4\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.081739501Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.082033712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.082042507Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.105763191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.105796847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.126964779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.127005299Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.147776211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.147810805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.167423624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.167455078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.186143323Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.186175837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.20586788Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.205902174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.224896932Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.224925582Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.243705971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.243732521Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.263564337Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.263603666Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.283395097Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.283420476Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.302498604Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.302531398Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.32110846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.321139164Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.340281344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.34030946Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.358609524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.358637585Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.378179556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.378211516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.397429133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.397464656Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.416518424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.416544618Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.435456277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.435482434Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.454395616Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.454427914Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.472659145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.472694724Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.472909622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.473024786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569418924Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569459564Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569465883Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569472531Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569477425Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569483687Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569489143Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569494862Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569518432Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569524188Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.56953091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569536535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569541579Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569546322Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569551226Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569556682Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569561838Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569565307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569572029Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.5697597Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"513c2a99-afe6-4ec4-a4c1-719ebdbdb199","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized)]","latency_us":486559,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:50.569920803Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"296.191899ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"513c2a99-afe6-4ec4-a4c1-719ebdbdb199\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"513c2a99-afe6-4ec4-a4c1-719ebdbdb199\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.867963831Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.868278517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.868286702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.89211846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.892165598Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.912528244Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.912636206Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.931311488Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.931351379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.950915149Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.950955219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.968616993Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.968651278Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.988195906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:50.988241104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.006729495Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.006768037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.02601938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.026057133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.044875405Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.04497633Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.065519714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.065561761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.086210425Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.086252942Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.106158939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.106199674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.125222371Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.125254209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.144281367Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.144323067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.162773032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.162823236Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.181676848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.181726049Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.200122653Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.200161017Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.218168773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.218205271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.236250109Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.236285321Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.254067317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.25411573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.254344287Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.25446578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345349361Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345389862Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345395899Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345402265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345408405Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345413168Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345417837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345423256Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345427756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345433175Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345438931Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345444819Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345468584Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345473957Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.34547857Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345482245Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345486876Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345490438Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345494798Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.345546661Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"97409c3a-4931-4897-a918-01af55cbb1e2","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized)]","latency_us":476197,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:51.345699402Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"377.854128ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97409c3a-4931-4897-a918-01af55cbb1e2\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97409c3a-4931-4897-a918-01af55cbb1e2\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.427861285Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50557"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.42833798Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"2a4d40a0-30ac-4433-9cb2-a6b554c96be7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50557","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":402,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.724954986Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.725269532Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.725277004Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.747253596Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.747285772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.769628505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.769674818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.790098202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.790133847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.809692524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.809726322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.829017048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.829113129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.848325044Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.848364203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.866911829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.866941979Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.886307067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.886333054Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.904788477Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.904818112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.923870871Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.9239438Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.943661721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.943689781Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.963237244Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.96326341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.981426202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:51.981519169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.000025201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.000052483Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.018935492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.018961901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.036857761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.036890321Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.055336164Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.055363962Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.074216754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.074243201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.093049106Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.093076913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.111188829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.11122319Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.111446759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.111564436Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.21029368Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210348066Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.21035597Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210362185Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.21036871Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210376454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210382895Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210389467Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210395439Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210403342Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210409708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210417274Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210423339Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210444949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210452561Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210458843Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210466409Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210472231Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210483603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.210644603Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c9216834-1c72-4a6e-a07b-30ced4825b1c","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized)]","latency_us":484279,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:52.210817837Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"665.879486ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9216834-1c72-4a6e-a07b-30ced4825b1c\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c9216834-1c72-4a6e-a07b-30ced4825b1c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.879189777Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.879523138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.879530817Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.902273417Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.902320049Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.923328286Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.923373437Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.942662241Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.942703687Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.961252368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.961293327Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.980642279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:52.980695793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.001172837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.00128889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.021153135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.021191386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.041543712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.041589529Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.06101325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.061048763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.08117131Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.081211118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.10011605Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.100162645Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.119350141Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.119397992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.139497936Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.139532166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.157478806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.157517479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.175744483Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.17578594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.195102909Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.19515316Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.213311162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.213352357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.232237162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.23227771Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.250465577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.250504408Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.269510548Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.269567287Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.269768647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.269887683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363276979Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.36331627Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363328955Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363334083Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363337889Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363341734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.36334539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.36334988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363353724Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363357362Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363361102Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363364693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363368537Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363373234Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363376946Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363379805Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363383518Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363386359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.363398585Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.36344846Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"522264e8-747f-4064-89a6-71fe4646dbc0","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized)]","latency_us":482809,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:53.36359774Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"491.00349ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"522264e8-747f-4064-89a6-71fe4646dbc0\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"522264e8-747f-4064-89a6-71fe4646dbc0\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.857374824Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.857772421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.857780137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.881399351Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.881433487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.90334383Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.903366199Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.923872839Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.923903402Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.942888803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.942922806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.962175683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.962207426Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.980651301Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.980679979Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.999963356Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:53.99998969Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.01938621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.019410914Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.04046079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.040499172Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.05900962Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.059040971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.077232703Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.0772626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.097542982Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.097570836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.116909845Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.116942011Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.136650614Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.136677257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.154993626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.155019351Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.173616204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.173643663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.192861169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.192888319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.211969249Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.212009731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.231031981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.231059085Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.249530436Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.249561129Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.249831898Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.249939891Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344392643Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344425775Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344430819Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344436538Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.3444404Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.34444625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344451594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344467054Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344473494Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344478145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344484276Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344488795Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344493707Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344498488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344503026Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344507292Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344512963Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344517492Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344522124Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:54.344598465Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a0ae8ddb-036f-442c-8236-2f745d87a29e","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized)]","latency_us":485762,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:54.344751506Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.028123515s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0ae8ddb-036f-442c-8236-2f745d87a29e\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0ae8ddb-036f-442c-8236-2f745d87a29e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.37705662Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.377343275Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.377350448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.398518645Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.39855893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.417253181Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.417296129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.43576313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.435860405Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.455823222Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.455928423Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.474854986Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.474956504Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.492876679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.492918745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.511531788Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.51157016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.529575182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.529610742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.547900761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.547938683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.566392112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.566440787Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.584687613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.584725855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.601515683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.6015498Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.621874672Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.621913128Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.64108492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.641123171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.659625046Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.659661947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.677232034Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.677263835Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.695576469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.695617972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.713362226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.713399549Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.732036196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.73207501Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.750025287Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.750062515Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.750267438Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.750372721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841008048Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841050498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.84105917Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841065602Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.84107329Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841079562Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841085637Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841091768Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841099184Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841105053Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841109524Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841126166Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841132269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841140013Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841146219Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841152678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841159907Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841165841Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841173266Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:55.841261946Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"f81ef56f-244f-43aa-b421-d462571e12cf","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized)]","latency_us":462826,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:55.841419871Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"770.103588ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f81ef56f-244f-43aa-b421-d462571e12cf\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f81ef56f-244f-43aa-b421-d462571e12cf\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.614182091Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.61453754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.614544956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.636705062Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.636747007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.655964975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.656023044Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.672605806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.672645323Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.688966224Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.689004063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.707947714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.707988205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.727696786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.727744656Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.744392273Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.744430862Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.762655973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.762696952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.783160155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.783202897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.802573714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.802616089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.821935035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.821980899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.840315688Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.840354145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.859192032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.85923346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.877172006Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.877207782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.896226846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.896269353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.913650925Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.913694707Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.931756305Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.931787421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.948318389Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.948352431Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.965414475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.965455312Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.98317379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.983227857Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.983459424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:56.983603547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080412815Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080466526Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.08047247Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080478864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080484048Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080490536Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080495542Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080501861Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080506633Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080511367Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080517509Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080528805Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080533437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080538105Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080541612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.08055589Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080559434Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080564187Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080570347Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:57.080632729Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"721760f9-509f-4d58-9b95-9c6fcc0cf637","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized)]","latency_us":465047,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:57.080785788Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.654904416s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"721760f9-509f-4d58-9b95-9c6fcc0cf637\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"721760f9-509f-4d58-9b95-9c6fcc0cf637\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.740991989Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.741301613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.741312207Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.766290773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.766335108Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.78652995Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.786569983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.804657903Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.804693294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.82420739Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.824241244Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.844453682Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.844487949Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.865007363Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.865109464Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.884331516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.88439057Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.903260663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.903298145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.922800603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.922834551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.942197127Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.942230362Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.960126681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.960158987Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.980341959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.9804414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.998848098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:58.998886649Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.01816177Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.018205524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.039663533Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.039697705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.058320996Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.058351728Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.076299955Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.076331343Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.095963325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.09599759Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.115368862Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.115412045Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.135024805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.135074709Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.135281469Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.135406618Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229172457Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229227161Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229235805Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229243464Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.22926827Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229274701Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.22928073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229288071Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229295552Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229301627Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229309024Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229315005Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229321896Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229328919Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229336503Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229342888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229348194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229352141Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.22935861Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:22:59.229419764Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"879a5b99-512b-450c-8fa8-d75e68cba129","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized)]","latency_us":486991,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.229566654Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.538724529s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"879a5b99-512b-450c-8fa8-d75e68cba129\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:22:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"879a5b99-512b-450c-8fa8-d75e68cba129\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:22:59.482505867Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.263863657s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.782396424Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.782641162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.782647285Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.80458552Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.804624577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.825063939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.825097071Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.84417548Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.844217387Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.863096934Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.863141991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.879867472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.879897847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.903038906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.903072825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.924075417Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.924107978Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.94319764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.943228493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.96248866Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.962521622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.980542277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.980578624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.999106276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:00.999137131Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.018984761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.019015456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.036829806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.036862487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.058535999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.058574541Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.077100381Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.07713355Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.097244732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.097283948Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.115800136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.11583084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.134136306Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.13423762Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.153940627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.153971659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.173738363Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.173782848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.173996686Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.174127508Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270805674Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270856488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270863932Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270868488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.27087446Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270880508Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270888036Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270913789Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270921823Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270927692Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270935202Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270941287Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270948477Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270954421Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.27096013Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270965849Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270971727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.27097589Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.270980005Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:01.27105362Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"301bca94-e621-413c-a883-cafb58d3a0e3","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized)]","latency_us":487265,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:01.271231082Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.098479127s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"301bca94-e621-413c-a883-cafb58d3a0e3\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"301bca94-e621-413c-a883-cafb58d3a0e3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.373761162Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.374047593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.374057268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.395216115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.39525475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.413866919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.413904917Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.434087674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.434128138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.45296985Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.453010125Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.470247049Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.470281324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.489134874Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.489168907Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.506412893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.50644551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.524839576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.52487253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.544071014Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.544106798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.562885194Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.562933673Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.581692225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.581728198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.600080398Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.600119334Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.61976072Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.619796467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.638669207Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.63870488Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.658789151Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.658823136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.677723102Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.677762459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.696247455Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.696282818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.717002341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.717039259Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.7360991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.736138767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.758326269Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.758374233Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.758638266Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.758751217Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.85256376Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852617864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852624511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852630886Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852636473Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852641912Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852647359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852654155Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852659612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852666324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852671499Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.85269445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.85269865Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852704856Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852710163Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852713922Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852720156Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852725369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.852730159Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:02.852911081Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"eaee07e1-4e79-4d42-b291-64208496d8b4","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized)]","latency_us":477731,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:02.85305934Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.458665668s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eaee07e1-4e79-4d42-b291-64208496d8b4\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eaee07e1-4e79-4d42-b291-64208496d8b4\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.31660536Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.316986245Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.316995554Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.340558133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.340595239Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.363349763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.363399133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.383551965Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.383589493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.401922416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.401964652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.422661729Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.422703185Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.441992451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.442027889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.461252748Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.46128888Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.480681912Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.480715531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.500554974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.500586372Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.520004187Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.52004242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.539720203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.539756504Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.558366601Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.558407749Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.577817293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.577855975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.59686419Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.596904044Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.615483925Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.615519307Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.635113643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.635156975Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.655290226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.655326415Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.674905387Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.674945373Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.695539703Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.695581404Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.71352815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.713577051Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.713781636Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.713970463Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810227359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.81027582Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810283254Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810289629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810294776Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810301957Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810308707Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810316985Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810325771Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810332539Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810340742Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810347483Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810352274Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810357271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810360786Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.81037919Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810384309Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.8103903Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810395212Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:04.810451172Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"671ff238-71eb-436b-b585-583385fc185a","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized)]","latency_us":492355,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:04.810599852Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.441034064s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"671ff238-71eb-436b-b585-583385fc185a\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"671ff238-71eb-436b-b585-583385fc185a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.256378606Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.256671271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.256678714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.279655025Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.279712373Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.300376038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.300422474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.319754368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.31980823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.33991377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.339962924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.361797697Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.361844273Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.380305745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.380419512Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.400829282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.400875502Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.419548051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.419611042Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.440239847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.440290764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.460416481Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.460470472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.479347137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.479399629Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.500466993Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.500512068Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.519997504Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.520035126Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.539531626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.539574808Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.558960923Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.55900899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.578201036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.578244855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.596587158Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.596630331Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.614922792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.614967924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.634989679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.635095557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.657329491Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.657399955Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.657570208Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.657686797Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752607777Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752666147Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752683903Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752701791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752716004Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752722632Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.75272626Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752731622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752735326Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752740838Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752745752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752750645Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752761014Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752766058Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.75277097Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752774467Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.75277803Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752783233Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752788033Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:06.752814302Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3961e7f7-0ece-4443-8c47-0f1d0fb15da6","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized)]","latency_us":494985,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:06.752953982Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.491649492s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3961e7f7-0ece-4443-8c47-0f1d0fb15da6\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3961e7f7-0ece-4443-8c47-0f1d0fb15da6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.252318158Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.252614648Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.252632695Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.276761914Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.276815437Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.297583691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.297633191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.316952084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.317002485Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.33669289Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.336750275Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.354580871Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.354634235Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.372541142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.372586134Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.390599706Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.390642288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.410251753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.410301601Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.430662586Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.430714272Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.450434557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.450483364Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.468813047Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.468866278Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.490303986Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.490356496Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.509580134Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.509640622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.530094549Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.530150865Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.549325434Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.549372957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.571426233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.571480055Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.591859823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.591907796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.612491446Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.612537956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.632301162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.632350831Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.650316204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.650380264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.650554623Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.650665756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749714228Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749787176Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749796017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749802026Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749808224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749812264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.74981728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749821302Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749826561Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749839386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749844543Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749849343Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749863715Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749869153Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749872734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749876437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749879747Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749883244Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.749892403Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:09.749896219Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"67ed365d-4949-4281-a06d-1015ed2cdefc","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized)]","latency_us":496130,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:09.750052099Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.393275911s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"67ed365d-4949-4281-a06d-1015ed2cdefc\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"67ed365d-4949-4281-a06d-1015ed2cdefc\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.153776802Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.154117945Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.154127311Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.178129439Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.178174712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.19823247Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.198278464Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.217947529Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.217987832Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.236794656Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.236836019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.255614863Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.255659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.274261066Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.274307473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.292655588Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.292700523Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.311622032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.311661764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.330396293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.330435819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.349397551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.349437723Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.368458166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.368494551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.386905397Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.386948935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.406394608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.406432971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.426776821Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.426819966Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.445881324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.445928978Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.465847541Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.465891126Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.484775392Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.484817242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.505311293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.505351869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.524567542Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.524606346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.544467961Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.544535752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.544811708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.544927727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637350505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637408537Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637414912Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637422609Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637428281Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637434947Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637441378Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637445044Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.63744885Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.6374532Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.63745981Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637470563Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637476338Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637489256Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637493054Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637498314Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637507951Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637511495Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.637515226Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:13.637520476Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"63d30ce8-08ab-4b4b-acf6-37060fe5978f","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized)]","latency_us":482342,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:13.63765846Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"6.035674771s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63d30ce8-08ab-4b4b-acf6-37060fe5978f\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63d30ce8-08ab-4b4b-acf6-37060fe5978f\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:15.82582721Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.009742726s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.691197456Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.691549643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.691560593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.713337403Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.713461535Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.733875314Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.736231072Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.75427563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.754331065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.773169992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.773226609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.793323172Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.793376095Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.81231448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.812363399Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.831407015Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.831448275Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.849443864Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.849491068Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.867716915Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.867768009Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.886467078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.886510364Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.905758008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.905805456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.923096225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.923141104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.94158254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.94163056Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.960720241Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.960774542Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.980258218Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:19.9803089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.000451416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.000501207Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.024132472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.026556084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.043446941Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.043503174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.063251229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.06329999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.082806451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.08287084Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.083063265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.08318682Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178237331Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178299347Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178308235Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178316625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178324453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178339304Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178346186Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178354323Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178362733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.17837042Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178377892Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178385458Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178389854Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178396268Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.17840674Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178428668Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178433196Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178439299Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178445591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:20.178462991Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"929d51c6-1915-47c0-9e5c-43852bf1510a","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized)]","latency_us":485856,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:20.178616256Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.697048302s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"929d51c6-1915-47c0-9e5c-43852bf1510a\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"929d51c6-1915-47c0-9e5c-43852bf1510a\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:21.544291734Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63994"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:21.544471917Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"dcb51e02-5dce-4bd0-931d-a0c2f0865742","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63994","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.887897239Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.888842944Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.888854878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.911113079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.91116033Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.932074979Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.932129917Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.951619704Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.951671764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.970478132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.970523611Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.989242836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:23.989280065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.009397623Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.009445089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.03040691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.030453851Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.052601098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.052664418Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.071457145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.071499352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.090879852Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.090923775Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.109870147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.109921222Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.130077386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.130117896Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.15034257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.150456704Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.169123214Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.169173146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.187742523Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.187794855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.207633685Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.207680964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.2266756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.226727726Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.245451589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.245489924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.264662899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.26478283Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.283307696Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.283357693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.283577371Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.283702306Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378308801Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378370189Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378383212Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378389971Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378396365Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378401315Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378407193Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378412209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378417881Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378422728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378428269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378438741Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378443944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378447732Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378452569Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.37845675Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378460866Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378468057Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378472341Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:24.378533046Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"725dd4f0-48dd-432b-b65c-ad406eb6f6fb","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized)]","latency_us":489120,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:24.378676757Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.400452822s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"725dd4f0-48dd-432b-b65c-ad406eb6f6fb\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"725dd4f0-48dd-432b-b65c-ad406eb6f6fb\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.790149687Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.790505284Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.790513778Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.8139542Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.813997541Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.83428963Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.834396071Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.85305771Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.85309253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.873875719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.873913464Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.893056072Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.893096432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.911872313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.911910405Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.929894585Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.929934767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.948351999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.948386134Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.967069834Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.967108113Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.985674131Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:27.98570535Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.003729724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.003763016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.023760257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.023796847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.040607658Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.040651525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.059840125Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.059874456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.078758163Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.078796714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.099463438Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.099503977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.119374281Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.119406737Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.136919297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.136953263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.155382325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.155421505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.17349765Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.173537073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.173742623Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.173859849Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264681278Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264738073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264750974Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264757827Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264763199Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264769142Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264791155Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264796949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264809484Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.2648151Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264820678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264824503Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264829509Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264832865Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.2648382Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264841565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264846281Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.264851635Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.26485636Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:28.264916595Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2c4555be-d957-4a35-87ef-4c66bfafc75b","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized)]","latency_us":473325,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:28.265076892Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"4.293491023s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c4555be-d957-4a35-87ef-4c66bfafc75b\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c4555be-d957-4a35-87ef-4c66bfafc75b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.571315161Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.571590603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.571599443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.593589312Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.593647794Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.614676285Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.614721914Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.633868118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.63391189Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.653244581Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.653306495Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.673483051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.675841773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.692575504Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.692618338Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.711749605Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.71179845Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.730527907Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.730632229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.75037384Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.750413938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.772526088Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.772579414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.792017216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.79206382Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.811712981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.81175576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.830893425Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.830942514Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.850150677Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.850193006Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.869560673Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.869600199Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.888674803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.888726029Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.908382968Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.908427069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:32.917643802Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"5.443871209s"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.929150715Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.929191881Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.948506139Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.948546556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.967736983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.967792259Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.967980868Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:32.968104798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067156228Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.06722731Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067236302Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067242695Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067248077Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067254217Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067259252Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067265364Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067285145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.06729153Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067296734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067302828Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067314846Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067320978Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067325084Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.0673298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067333896Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067338603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.067343404Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3230","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:33.067404398Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c7372207-f9ec-4662-9042-cb417a83004c","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\", TimeNotSynchronized), (\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\", TimeNotSynchronized), (\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\", TimeNotSynchronized), (\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\", TimeNotSynchronized), (\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\", TimeNotSynchronized), (\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\", TimeNotSynchronized), (\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\", TimeNotSynchronized), (\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\", TimeNotSynchronized), (\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\", TimeNotSynchronized), (\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\", TimeNotSynchronized), (\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\", TimeNotSynchronized), (\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\", TimeNotSynchronized), (\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\", TimeNotSynchronized), (\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\", TimeNotSynchronized), (\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\", TimeNotSynchronized), (\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\", TimeNotSynchronized), (\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\", TimeNotSynchronized), (\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\", TimeNotSynchronized), (\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\", TimeNotSynchronized)]","latency_us":494694,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:33.06761314Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"12.134956743s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7372207-f9ec-4662-9042-cb417a83004c\", \"content-length\": \"124\", \"date\": \"Tue, 10 Sep 2024 21:23:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c7372207-f9ec-4662-9042-cb417a83004c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 19b992ad-ed77-421f-ac97-09b7e8b99397, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 117a78d0-27bb-4004-9e68-77c6e39f71da, kind: Service { id: 19b992ad-ed77-421f-ac97-09b7e8b99397 }, name: Name(\"ntp-19b992ad-ed77-421f-ac97-09b7e8b99397\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 116])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b7df98df-a73e-420a-ba60-6f5404c39291, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2229031-29c9-471e-b908-4fd2e6787eb8, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 22ec8121-6583-4054-9535-d2454e5c4dec, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: efb05780-75dc-4711-84b3-13f5e729e36e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 19a73de5-6a55-413b-9592-dba813ea235c, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 392d3075-49c0-4f37-8684-de3f94e05aed, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: be5c1118-d282-446a-9683-cce99d6e0366, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28b420c6-86bc-4082-831d-c896c94357dd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62adf1a2-944b-44c7-83d4-0b091ec90757, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: a8dfcb2a-a446-4401-92ac-04593f5a0646, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f593b8d9-1792-49e8-9788-43f4e29c8af2, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External } } } }, OmicronZoneConfig { id: 60322f13-f2a6-4667-8db9-6133275251a4, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External } } } }, OmicronZoneConfig { id: af4d562d-aa30-4543-9fd8-acc679264a47, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External } } } }, OmicronZoneConfig { id: 85653e06-ae88-49fe-98a6-f316466bd4a9, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External } } } }, OmicronZoneConfig { id: b620bc71-86f0-40bd-9dae-6f5872132134, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External } } } }, OmicronZoneConfig { id: b462ab5a-f98e-449e-932e-a439e4629afa, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.236776864Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.237047684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.237053703Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 48d3ebb5-3dc4-4b69-a505-4bb07f9482ea, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.260383493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.260503702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b8ae7144-e4b7-44d2-8d80-34bfd3575616, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.28088795Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.280922816Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 94337c1c-5cb5-4050-bd90-d4c50dd633f1, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.299059248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.299097798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b7df98df-a73e-420a-ba60-6f5404c39291, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.317869272Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.317907166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 61d20555-41e5-4532-a1c0-bfd0c4966c8d, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.339748219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.339782279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 71834d57-8f47-4ba6-a6ab-2f8adc6a35e3, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.358764179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.358805486Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 9dddae67-2068-4cf1-b0cb-d2382c7eda0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.380367643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.380403296Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: a2229031-29c9-471e-b908-4fd2e6787eb8, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.400527513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.400569814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 22ec8121-6583-4054-9535-d2454e5c4dec, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.421813045Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.421923874Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: efb05780-75dc-4711-84b3-13f5e729e36e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.44247095Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.442508751Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: be5c1118-d282-446a-9683-cce99d6e0366, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.462442874Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.462476043Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 628c6ac7-c011-44fd-b30d-3d53d9bab46b, dataset_name: DatasetName { pool_name: ZpoolName { id: 0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.481550502Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.481581609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: f593b8d9-1792-49e8-9788-43f4e29c8af2, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d09a0b5-64fe-49da-9f0c-2aeb05367248 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.502858065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.502893719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 60322f13-f2a6-4667-8db9-6133275251a4, dataset_name: DatasetName { pool_name: ZpoolName { id: b41c1bbc-10c8-46e3-9c3a-722fb6b38e83 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.522750165Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.522792129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af4d562d-aa30-4543-9fd8-acc679264a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 233394d8-3133-44e6-a8c8-baa69c3f73c6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.543011567Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.543069562Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: c50f8de4-26d4-43c0-ab4e-44cd5b658ed3, dataset_name: DatasetName { pool_name: ZpoolName { id: a1f44de4-bbe9-4024-862c-8b1b1210d80a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.564072806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.564107897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 85653e06-ae88-49fe-98a6-f316466bd4a9, dataset_name: DatasetName { pool_name: ZpoolName { id: eb1402a5-e990-4914-9b22-a55b7b60e97b (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.586137412Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.58617315Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b620bc71-86f0-40bd-9dae-6f5872132134, dataset_name: DatasetName { pool_name: ZpoolName { id: ea86afc8-d62a-4b61-9643-afc4d22f0535 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.605769892Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.605807617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 17ffab08-2d84-4a34-a5cc-509075bfaa18, dataset_name: DatasetName { pool_name: ZpoolName { id: e82c3049-23d1-4237-ad1f-14e37401e51a (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.625972958Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.626009024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: b462ab5a-f98e-449e-932e-a439e4629afa, dataset_name: DatasetName { pool_name: ZpoolName { id: 099b9246-cdfd-4461-86ab-695e2d382672 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.646264744Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.646309286Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.646595283Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.64671776Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838048609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838110908Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838117189Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838121905Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838125589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838128973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838138573Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838145548Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.83814928Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.838152514Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.852474993Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.852717604Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.852734329Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.864418843Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.865971497Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.877435511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.892470749Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.892615624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:45.892630146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.165417823Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.170000355Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.174655114Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.179016994Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.183772218Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.188871794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.193771668Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.198774849Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.204079075Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.209072631Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.213659879Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.218480094Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.223103158Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.227739184Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.232643265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.237614171Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.242156263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.247590009Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.252748872Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.257828054Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.26382251Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.268752894Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.273917276Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.280415021Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.285654791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.338539888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.391528674Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Installing Omicron zone: oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.391780454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.401879486Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.411935598Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.523925371Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.614674888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.710987566Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.715995767Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.716030454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.829319925Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 186, 179]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.829378097Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.833310399Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.833323853Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.900598666Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 152, 96]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.900618719Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.90305681Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.903075241Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.957900079Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 202, 234]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.957928045Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"nexus"}
{"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.960120318Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.960129421Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.960140981Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:46.960147206Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.012052882Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 139, 222]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.012083389Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"}
{"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.012203293Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.012212902Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.014441068Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.014473057Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.068950043Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 215, 6]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.068979669Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"external_dns"}
{"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.069860676Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.07154017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.071552845Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.074624642Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.668720676Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.668774827Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"Configuring new Omicron zone: oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.68870193Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.736689394Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.736750333Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.73677632Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.736783418Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.736940226Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.739696817Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.739942025Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Configuring new Omicron zone: oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.744918278Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.800859823Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.810818266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.810853966Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Installing Omicron zone: oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:47.901481653Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:48.400663608Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:49.370996945Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:49.684496074Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:49.842162811Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:49.889349581Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:49.908617261Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:49.915804754Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:54.530982752Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"5.868768086s"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:54.967070905Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36078"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:54.981550293Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c2b10bb2-1952-4ddf-b07d-9da047fc4292","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36078","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7214,"response_code":"200"}
{"msg":"Profile for oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService10'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::10'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::10'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.17105489Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.173311486Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService17'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::11'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::11'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.212516922Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService20'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::f'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::f'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.275656442Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.279083143Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.2941354Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService12'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::18'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_ea86afc8-d62a-4b61-9643-afc4d22f0535/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::18'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='2fdf094b-e6a9-4ab7-93b9-405184186899'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.322714972Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService14'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::19'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_e82c3049-23d1-4237-ad1f-14e37401e51a/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::19'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='8273dc44-e48a-41dc-b0e9-7208d259ed81'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.325531499Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService15'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::13'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_1d09a0b5-64fe-49da-9f0c-2aeb05367248/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::13'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='63497241-a030-4bff-921a-3da6c9e98880'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.330221926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService19'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::16'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_a1f44de4-bbe9-4024-862c-8b1b1210d80a/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::16'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='7a76c92a-fcce-4bb9-8a86-ad677274f065'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.3484637Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.348869318Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService13'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1a'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_099b9246-cdfd-4461-86ab-695e2d382672/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::1a'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='a149f281-2c07-4d42-95c4-41a10f68245f'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.353974489Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.35398768Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.382056672Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.395608191Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService11'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::14'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_b41c1bbc-10c8-46e3-9c3a-722fb6b38e83/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::14'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='f3409eeb-e706-4318-88b9-8717feb2322c'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.432713328Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.432794752Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.448863912Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService16'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::17'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_eb1402a5-e990-4914-9b22-a55b7b60e97b/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::17'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='546a7e98-4ffd-4b20-a20a-5e1a42747481'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.603485649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.603737522Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService18'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::15'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_233394d8-3133-44e6-a8c8-baa69c3f73c6/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::15'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='1620aaf6-8840-430d-ad6e-7c5760eff4f5'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.617201444Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.617468593Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService21'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::12'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::12'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='e2ff4183-78c3-4d9d-8d93-e1d7e83aadf2'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.684226076Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:55.684388585Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 54.382213ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.75046755Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 35.062756ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.766430119Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 29.668566ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.782377889Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 70.314917ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.797388815Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 56.528108ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.818800517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 25.27402ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.828858565Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 43.968422ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:55.843641053Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 44.616848ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.07985471Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 35.196133ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.227410468Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 89.861778ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.312092227Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 57.405247ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.357640274Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 64.058093ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.403827836Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 112.809525ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.4587634Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 63.253625ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.484926499Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 59.278445ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.637976807Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 120.810466ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.671936712Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 71.835504ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.755652244Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 65.751722ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.79651834Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 78.639406ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.846286766Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 58.221957ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.8796592Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 69.410167ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.966736258Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 287.415175ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.983255267Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 124.904434ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:56.997549279Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 132.706397ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.008779067Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 141.000085ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.02494665Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 108.538287ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.03584793Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService22'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::d'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value='392d3075-49c0-4f37-8684-de3f94e05aed'/>\n        <propval type=\"astring\" name=\"address\" value='[fd00:1122:3344:101::d]:12223'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:57.036377049Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 133.999478ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.050727507Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 251.179822ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.065539769Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 168.28848ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.078199016Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 160.769779ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.092316069Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 277.486154ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.107382959Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 207.647499ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.119918162Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 162.984601ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.133184724Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 208.742593ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.145009726Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 212.025477ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.16168241Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 122.649129ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.173994827Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:57.174161219Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 574.729039ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.193421541Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 449.924192ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.207024681Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 579.021815ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.259524839Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 424.105303ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.272561384Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 250.267349ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.282837541Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 545.625747ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.308029619Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 518.739022ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.317909221Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 414.44736ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.328754485Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 218.510661ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.341375837Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 433.139005ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.367343424Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 400.924233ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.386674301Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 467.504018ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.396948948Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 58.340168ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.414369066Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 60.312022ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.484871301Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 447.55494ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.548590231Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 216.74599ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.559001653Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 1.148553621s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.572564933Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService27'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::9'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte4'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.1.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.1.6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::9]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='172.30.1.6:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:57.573150558Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:57.573380128Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 1.162219825s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.672467365Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 468.991046ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.710863692Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 1.192325628s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.762748291Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 727.61249ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.78201295Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 394.664341ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.794968489Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 921.538769ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.807962065Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 586.310968ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.821010195Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 64.891008ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.831328805Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 654.257418ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.850153149Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 461.359933ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.862882182Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 689.47474ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.873637104Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 408.7692ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.888069759Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 55.983334ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.909211221Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 269.804362ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:57.978363343Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 1.431197357s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.012786234Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 624.54644ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.220769084Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 877.345983ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.269734051Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 513.980548ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.315057678Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 1.259222748s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.38701656Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 637.36391ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.469505708Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 1.409284966s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.542446347Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 1.02381843s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.616137648Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 855.715176ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.674543638Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 1.307189408s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.695540638Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 1.28570121s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.742259416Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 1.214647928s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.759172746Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.064190606s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.846969413Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 553.472527ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.861998915Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 590.207844ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.877317178Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 1.242312903s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:58.974744481Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 1.100251723s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.124853125Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService28'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::8'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte5'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.1.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.1.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::8]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='172.30.1.5:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:59.150304109Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:59.150668164Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 1.275703073s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.183366952Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService24'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::c'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte1'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.7'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:59.397427323Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:59.398545546Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService25'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte2'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:59.408043842Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:23:59.408282459Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 696.60664ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.435101532Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 693.888591ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.467280258Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 782.82983ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.490824684Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 34.706234ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.506758591Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 509.921422ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.552151513Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 75.380705ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.569685102Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 601.538922ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.710587053Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 1.301475811s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.757164451Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 107.765257ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.871230011Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 33.630998ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.885005206Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 44.050162ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.898949832Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 865.924979ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.932432623Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 101.083848ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.958435507Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 104.766998ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.974136669Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 1.226664073s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:23:59.991606421Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 568.556833ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.020185909Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 219.6051ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.049020641Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 1.470316499s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.069682416Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 1.35846159s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.087990783Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 1.036483186s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.111802451Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 276.183735ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.132451422Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 246.696725ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.147795066Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 1.278421546s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.168024668Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 717.081986ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.188845095Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 514.78905ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.235338574Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 913.063314ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.250484607Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 982.182272ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.284172398Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 1.240825519s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.300750032Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 1.401292462s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.327354474Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 521.199611ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.421351314Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 535.764428ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.438171369Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService26'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::a'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte3'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:00.457615653Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:00.457743483Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 1.161461253s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.483429546Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 790.656153ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.602851628Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 52.463247ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.723046039Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 562.133647ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.766801988Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 61.876261ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.788088371Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.088554714s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.819323155Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 199.420158ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.916037582Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 527.78764ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:00.971994413Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 884.211661ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.019278168Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 1.170453349s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.032743653Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 761.81359ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.079611931Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 498.399008ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.133225776Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 779.233249ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.165271445Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 1.093466721s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.184836996Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 588.104654ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.235588492Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 826.574724ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.281960871Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 1.107219995s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.348469192Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 1.242373068s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.407782677Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 1.041156706s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.459351537Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 622.269307ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.471143642Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 1.287265149s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.51170471Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 1.473226864s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.558861823Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 1.097273189s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.577406016Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 1.065867921s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.644846583Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 669.9512ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.661199148Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 827.771069ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.746072769Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 1.232046321s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.840482857Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 665.420316ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.854399179Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 916.068671ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.918535838Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.103562539s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.930878113Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 1.022786829s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:01.957306674Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 1.152239361s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.11182553Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 952.051131ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.125299596Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 685.561792ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.217629936Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 1.049918428s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.29176276Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 1.312932988s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.347774197Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 1.403830885s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.469490161Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 611.919645ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.515117677Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 1.223919308s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.532196195Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 701.279255ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.587674373Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 1.302543656s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.664404062Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 1.447404531s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.687637917Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 539.808104ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.7267388Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 939.671574ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.813465703Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 1.260539591s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.851736871Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 1.223043863s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.918557916Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 1.40206081s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:02.999926226Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 1.444720649s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.048494563Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.092901806s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.062996557Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 983.464999ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.086624939Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 1.123755791s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.099505694Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 938.765884ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.143280217Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService23'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::e'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::e'/>\n        <propval type=\"astring\" name=\"listen_port\" value='8123'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::e]:8888'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:03.155944468Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:03.156283104Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 1.480983457s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.285074292Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 751.935093ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.30161321Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 1.210504012s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.320181304Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 1.470300278s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.358551694Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 37.057651ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.467519625Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 133.499207ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.519880129Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 249.069287ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.670390315Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 1.179636785s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.693503268Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 1.087408808s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.835043858Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 686.762722ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.858886854Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 1.216013403s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.895635315Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 437.674475ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.945074778Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 1.091138114s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:03.985176928Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 1.227158969s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.074859142Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 1.466648431s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.089560139Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 1.461966197s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.105768992Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 1.478933968s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.125706909Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 1.025800639s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.148381525Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 1.444213112s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.160008778Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 734.31433ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.179997127Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 1.404916559s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.238369355Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 716.269093ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.397878689Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 852.11483ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.414136062Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 1.387302275s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.509018428Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 1.094141281s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.546053034Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd\") failed: Property not found. retry in 1.231459675s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.559330336Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 722.732252ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.780448444Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134\") failed: Property not found. retry in 1.394671852s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.841699283Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 1.477736124s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.888330846Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.244924073s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.930273078Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646\") failed: Property not found. retry in 921.204451ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:04.941221369Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3\") failed: Property not found. retry in 1.028019719s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.087824827Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4\") failed: Property not found. retry in 1.148175978s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.123332727Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.30862968s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.135386507Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 586.145048ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.186599531Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 1.102465244s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.280373777Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 1.091242698s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.316720341Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 595.517376ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.517849611Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47\") failed: Property not found. retry in 715.348086ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.569872919Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 1.401562925s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.581605713Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 669.885737ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.618381685Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 1.369599959s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.630363094Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18\") failed: Property not found. retry in 897.754342ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.653834813Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 1.371541416s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.664956147Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2\") failed: Property not found. retry in 653.169319ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.78658898Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa\") failed: Property not found. retry in 1.32469092s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:05.913491414Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9\") failed: Property not found. retry in 845.508067ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.132475285Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 772.639209ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.190010489Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 974.47637ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.350693083Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 711.975687ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.38549708Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 1.206297394s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.411603276Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 891.306008ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.429716602Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.204707004s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.487164674Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 709.523575ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:06.98210694Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 1.115187653s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.041893303Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 1.046414459s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.066088015Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 692.653166ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.088265522Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 721.300199ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.118119413Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 523.16318ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.34257512Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 1.232836753s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.375578726Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 612.781591ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.641567227Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.16610049s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.711198322Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.292444426s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.727284152Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 678.160021ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.803437221Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 927.77291ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.858183547Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 1.100986146s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:07.88817497Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 993.2827ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.130157082Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 1.460530808s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.179192488Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 867.933387ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.272945943Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757\") failed: Property not found. retry in 771.172282ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.500061676Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 1.2214508s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.62632171Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed\") failed: Property not found. retry in 1.421611992s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.803366489Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e\") failed: Property not found. retry in 1.288255558s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:08.892156268Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 884.626904ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.00749646Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 611.57156ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.038295287Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec\") failed: Property not found. retry in 830.050576ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.137324846Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b\") failed: Property not found. retry in 1.464960901s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.157733192Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e\") failed: Property not found. retry in 835.907467ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.654078223Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.341676537s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.66656387Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c\") failed: Property not found. retry in 905.824164ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.859855383Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\") failed: Property not found. retry in 1.40007369s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:09.905114052Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:10.831041249Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"23.165936174s"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 615.093227ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:11.02311056Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:11.50469092Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.388664712s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:11.649109601Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:11.939884738Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:12.064093757Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.498850579s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:13.048323252Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:13.068810919Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:13.906561276Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:14.072157045Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.238340953s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:14.557188579Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:15.085959674Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.243453358s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:15.80669883Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:15.907716792Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:16.0967071Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 700.511697ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:17.060491246Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:17.100756709Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 680.61063ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:17.77196725Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:17.909170193Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:18.112177676Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 878.082286ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:18.463085389Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:19.116731254Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.228805637s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:19.351274257Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:19.907796108Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:20.119520333Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 798.255071ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:20.590305969Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:21.130855611Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"wait for service Some(\"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366\") failed: Property not found. retry in 1.252011132s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:21.400282323Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:21.908551992Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:22.134422244Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690152172Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_pantry_62adf1a2-944b-44c7-83d4-0b091ec90757"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690219453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_60322f13-f2a6-4667-8db9-6133275251a4"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690230103Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_b620bc71-86f0-40bd-9dae-6f5872132134"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690236094Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_b462ab5a-f98e-449e-932e-a439e4629afa"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690241744Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_17ffab08-2d84-4a34-a5cc-509075bfaa18"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690246284Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_f593b8d9-1792-49e8-9788-43f4e29c8af2"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690252044Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_85653e06-ae88-49fe-98a6-f316466bd4a9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690257314Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_pantry_a8dfcb2a-a446-4401-92ac-04593f5a0646"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690262944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_af4d562d-aa30-4543-9fd8-acc679264a47"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690306435Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_c50f8de4-26d4-43c0-ab4e-44cd5b658ed3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690315645Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_pantry_28b420c6-86bc-4082-831d-c896c94357dd"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690327495Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_crucible_628c6ac7-c011-44fd-b30d-3d53d9bab46b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690334756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_nexus_19a73de5-6a55-413b-9592-dba813ea235c"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690341976Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_external_dns_efb05780-75dc-4711-84b3-13f5e729e36e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690363726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_nexus_9970237b-c8a0-4fe2-a597-d8dfdeab350e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690369296Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_external_dns_22ec8121-6583-4054-9535-d2454e5c4dec"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690375066Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_oximeter_392d3075-49c0-4f37-8684-de3f94e05aed"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690637281Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_clickhouse_be5c1118-d282-446a-9683-cce99d6e0366"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.690644261Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_nexus_1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.692207668Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.702845024Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.71698401Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"fb8e1813-c9eb-4e1c-96d4-08143fdfa9fe","remote_addr":"[fd00:1122:3344:101::1]:52333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37439801,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.738403904Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1322"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.738530506Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.738822171Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.742465515Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:707"}
{"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.744382639Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"}
{"msg":"rack_network_config: RackNetworkConfigV2 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.2.174,\n    infra_ip_last: 10.151.2.174,\n    ports: [\n        PortConfigV2 {\n            addresses: [\n                UplinkAddressConfig {\n                    address: V4(\n                        Ipv4Net {\n                            addr: 10.151.2.174,\n                            width: 24,\n                        },\n                    ),\n                    vlan_id: None,\n                },\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            lldp: None,\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Net {\n                            addr: 0.0.0.0,\n                            width: 0,\n                        },\n                    ),\n                    local_pref: None,\n                    nexthop: 10.151.2.100,\n                    vlan_id: None,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Net {\n        addr: fd00:1122:3344:100::,\n        width: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.779155946Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.784867305Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.902400796Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:22.975705355Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:23.105040312Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:23.13764911Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:23.290494387Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:23.513955607Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:23.808514006Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:23.906444544Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:23.963723734Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:24.141597047Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:24.275199459Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:24.675830279Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:25.063816738Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48515"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:25.068225255Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"fa6f60d0-45aa-40be-a367-50a42990d599","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48515","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":692,"response_code":"200"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:25.14472692Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:25.408462532Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:25.905960162Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:26.14713581Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:26.183651017Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:26.837016268Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:27.150536638Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:27.500491174Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:27.906815974Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:28.152932899Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:28.892757778Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:29.156171455Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:29.906164171Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:30.159568233Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:30.351104536Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:31.163939159Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:31.906225241Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/403254df-bc94-46d0-9b33-45b17d93603a): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-09-10T21:24:32.167923168Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Failed to handoff to nexus: Error Response: status: 403 Forbidden; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8673fc7-6e3d-4f62-8bce-64c7583e4342\", \"content-length\": \"113\", \"date\": \"Tue, 10 Sep 2024 21:24:32 GMT\"}; value: Error { error_code: Some(\"Forbidden\"), message: \"Forbidden\", request_id: \"f8673fc7-6e3d-4f62-8bce-64c7583e4342\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:32.602389949Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:24:34.035661896Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9f8aa516-8c85-4db9-b7e9-d0cd26f32925\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Tue, 10 Sep 2024 21:24:34 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"9f8aa516-8c85-4db9-b7e9-d0cd26f32925\\\" }\"","delay":"23.413000368s"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"524d3350-6cfe-4d2a-a70a-3e0b2dc7364f\", \"content-length\": \"173\", \"date\": \"Tue, 10 Sep 2024 21:24:33 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"ebaa0c37-a4ae-43ac-95a9-ac9cdaefcd75\\\"\", request_id: \"524d3350-6cfe-4d2a-a70a-3e0b2dc7364f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:34.219545815Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.066629303Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.16392542Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42593"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.165913155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430389Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"ebc8806d-abcb-4e4f-83d8-c17708ae74c8","remote_addr":"[fd00:1122:3344:101::c]:42593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":266425,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430625884Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60231"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430662445Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60081"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430673955Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42150"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430707196Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38447"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430795917Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47425"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.430809337Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56224"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.431665402Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"72a956f8-435e-4da2-916e-ef32a84c7afc","remote_addr":"[fd00:1122:3344:101::a]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":894,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.432034089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.432049929Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8153f9c4-f8ec-4fae-adc2-5e89c356d099","remote_addr":"[fd00:1122:3344:101::b]:56224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1113,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.432067729Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.441813229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.442371559Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"43a62a14-df8f-4010-aa28-0ef5282e0674","remote_addr":"[fd00:1122:3344:101::c]:38447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":11410,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.443664382Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d895f029-7f88-4c22-aa8e-3b4a459a9911","remote_addr":"[fd00:1122:3344:101::a]:60081","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":12799,"response_code":"200"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.443788174Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.446893408Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.456924813Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"d613c1d5-e45a-4878-a28e-e2d6e439c916","remote_addr":"[fd00:1122:3344:101::c]:42593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":20294,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.470761214Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53489"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.476045157Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"54b3e91b-9050-4f5e-942e-c56fea44667f","remote_addr":"[fd00:1122:3344:101::b]:53489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":605,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.571707096Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d10c632c-3ae3-4850-be98-58820561e4b4","remote_addr":"[fd00:1122:3344:101::a]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8369,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.610547324Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"45e518e3-2df7-4aa1-ac90-244b768fb86a","remote_addr":"[fd00:1122:3344:101::b]:56224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5760,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.687372114Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"0cf7fafc-1acf-48f7-8a34-634d3a4d7b75","remote_addr":"[fd00:1122:3344:101::a]:47425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":256194,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.688004245Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.688628676Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.689031123Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.689253947Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"e621d4e3-1ef0-4c66-8b86-254a0140d7de","remote_addr":"[fd00:1122:3344:101::a]:47425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1298,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.694485298Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"b4f51e83-67f5-4df0-9cff-c53102e4d65f","remote_addr":"[fd00:1122:3344:101::b]:42150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":263575,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.695705899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.695758611Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.696023315Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:36.69631611Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"a9f068a2-91aa-4bc6-8f65-188a0b9f3aa8","remote_addr":"[fd00:1122:3344:101::b]:42150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1271,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.404977015Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44438"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.405207949Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.623428867Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"0f7a56d2-58d5-40c8-b544-373e23d6b1fe","remote_addr":"[fd00:1122:3344:101::c]:44438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":218341,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.62357172Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63202"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.623968796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.624014527Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.624381663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.624637868Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"66c55ba7-2315-4d59-9315-f1a95726db16","remote_addr":"[fd00:1122:3344:101::c]:44438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":800,"response_code":"200"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.63107166Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631262504Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631442317Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631459367Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631471817Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631492058Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631501078Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.631533918Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0c19d471-e658-47fa-9979-761b3fcca623","remote_addr":"[fd00:1122:3344:101::c]:63202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7857,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:37.633121496Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.023612949Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37970"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.023747322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.225921289Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"dbbaa218-73cf-4f6f-9186-0929371ccd1d","remote_addr":"[fd00:1122:3344:101::c]:37970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":202214,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.226376427Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.226434888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.226699693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.226901936Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"4b77937a-7a86-4e76-a336-03d48c419c1c","remote_addr":"[fd00:1122:3344:101::c]:37970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":600,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.509494857Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49330"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.509793243Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.510021296Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.51021806Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.51023934Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.510253631Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.510272681Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.510291221Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.510329882Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"86afb16d-d347-4b56-8803-930f7d98d1f0","remote_addr":"[fd00:1122:3344:101::c]:49330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":752,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.783865575Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41111"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784108849Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.78415266Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784192071Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784214371Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784226761Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784247072Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784269232Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.784287132Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0a0ec5cd-9abc-4284-a8db-d20592f3df6a","remote_addr":"[fd00:1122:3344:101::a]:41111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":363,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871694747Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51766"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871880241Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871916641Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871944182Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871964982Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871988752Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.871997812Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.872009563Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:38.872026703Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"158b7310-4562-40a2-841c-1af75b1d3568","remote_addr":"[fd00:1122:3344:101::b]:51766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":271,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:47.469376775Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63277"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:47.469447626Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"77b3e8b1-30b1-40de-b8c8-4fc29e7dff0a","remote_addr":"[fd00:1122:3344:101::c]:63277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:47.478993393Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"55107228-273f-4cda-aa18-0fb088c98b14","remote_addr":"[fd00:1122:3344:101::c]:63277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:55.106028957Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55847"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:24:55.106730209Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"7bb402c5-3663-4f8e-9aa2-2a12ac326ac2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55847","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":506,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.08530731Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42850"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.085398462Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"33b492f3-0ea1-42ec-b664-5686689b8c35","remote_addr":"[fd00:1122:3344:101::c]:42850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.100365013Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"3d77a429-7499-4e22-bdad-72c165232670","remote_addr":"[fd00:1122:3344:101::c]:42850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.24289197Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60517"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.242958481Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e63d355a-6be5-42e7-9ae7-a8101ccc8b74","remote_addr":"[fd00:1122:3344:101::a]:60517","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.248673141Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52536"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.249560276Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f45ce56f-7928-4ec6-aaac-537361fbb939","remote_addr":"[fd00:1122:3344:101::b]:52536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":851,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.253371723Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40077"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.254169657Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.34774649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.40793278Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"832158e7-19ff-40c9-b709-2adacf099fc7","remote_addr":"[fd00:1122:3344:101::c]:40077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":154510,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.408314416Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6c189c38-6cd4-4642-8ae0-3ad268162a20","remote_addr":"[fd00:1122:3344:101::b]:52536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":205,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.408399168Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64737"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.408429318Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45041"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.408461669Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50194"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.413515487Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f1b25d8c-54db-48d6-b078-c95cabfecd47","remote_addr":"[fd00:1122:3344:101::a]:60517","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4993,"response_code":"204"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.41369854Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.41370637Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"a18f66f7-c4de-4547-b26a-13df25e09161","remote_addr":"[fd00:1122:3344:101::a]:50194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5112,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.413782942Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"e3ccea78-82d8-4c0c-aa9b-4c5f9029344a","remote_addr":"[fd00:1122:3344:101::c]:45041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5234,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.418766219Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4d4a94b4-5db2-4410-89af-b6cafb24f4c3","remote_addr":"[fd00:1122:3344:101::c]:40077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":10315,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.453848491Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.461993903Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.462355299Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.463462929Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.464379385Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.464402535Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"05725206-8334-4dd4-8a75-3c3fac3844e2","remote_addr":"[fd00:1122:3344:101::c]:40077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5648,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.513727566Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35546"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.514668922Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"80d45b71-4cfc-4608-9b69-31b5382af133","remote_addr":"[fd00:1122:3344:101::b]:35546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":911,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.520258029Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"799f0771-a5e1-417e-8bb8-f43c47646979","remote_addr":"[fd00:1122:3344:101::a]:64737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111612,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.520765638Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6c5d90e2-52f6-436e-928c-438c041dd90e","remote_addr":"[fd00:1122:3344:101::a]:64737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":239,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.537228956Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45729"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.537355038Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.572516581Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.624646881Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1ebda057-4c34-4a17-a148-f1f6d4bfaf22","remote_addr":"[fd00:1122:3344:101::b]:45729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87348,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:06.625552467Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a3fb63f3-7015-43f5-9b68-42f0a23612a1","remote_addr":"[fd00:1122:3344:101::b]:45729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":633,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:25.141171867Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42288"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:25.141241128Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"4ba00617-48a6-4a6e-95eb-23a4b3ce834d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42288","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:27.591203665Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:60045"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:27.638238056Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"7d3af383-4aa8-42b9-b49f-b69124121b57","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46690,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.137354118Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61653"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.13751237Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"cf48af6d-f020-432c-ba83-9fe3b16f7345","remote_addr":"[fd00:1122:3344:101::c]:61653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.26067531Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64416"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.260789272Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6aa602c9-d370-4596-a0ce-b003327877d6","remote_addr":"[fd00:1122:3344:101::a]:64416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":82,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.280398754Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a1db5286-c1b4-4a20-be73-fbdabeaae13e","remote_addr":"[fd00:1122:3344:101::a]:64416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.28419793Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49944"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.284349132Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"e8e6b547-1c25-44d3-ac99-ba113dc212f8","remote_addr":"[fd00:1122:3344:101::c]:49944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.299171481Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"72f428ed-8517-40e8-be23-456a3b32b3ba","remote_addr":"[fd00:1122:3344:101::c]:61653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1030,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.381493138Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49264"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.38160166Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.426713667Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.492911742Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"222a8f57-db2d-40ab-91ce-243b67ffd64c","remote_addr":"[fd00:1122:3344:101::a]:49264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111366,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.493244297Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47122"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.493292978Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59378"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.493580153Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"85f82e95-d46a-4cbe-a1a7-207df4f01592","remote_addr":"[fd00:1122:3344:101::a]:49264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":330,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.493590514Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bbb57f61-c96c-4db5-9a92-e49e42ed0509","remote_addr":"[fd00:1122:3344:101::b]:59378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":189,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.494681522Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.53978328Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.568212056Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38904"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.56846751Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"52895aa4-dd49-4005-9e0e-820749c61f8c","remote_addr":"[fd00:1122:3344:101::a]:38904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":191,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.60110347Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"88656b9e-5532-4c3f-8b3a-d84daaacbb09","remote_addr":"[fd00:1122:3344:101::c]:47122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107688,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.601615948Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6c020773-3c58-43f5-9cfd-5067acaebb49","remote_addr":"[fd00:1122:3344:101::c]:47122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.633374433Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53870"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.634303069Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"4d66a9e3-7571-46d9-a1f7-af3607f68b85","remote_addr":"[fd00:1122:3344:101::b]:53870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":880,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.637011386Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"547d9096-6378-4713-b313-2ea9cf9e170c","remote_addr":"[fd00:1122:3344:101::b]:59378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.825890812Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59787"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.825978263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.866146734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.928123855Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"97c1e970-6868-454e-be49-18826d753355","remote_addr":"[fd00:1122:3344:101::b]:59787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102178,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:36.928466191Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f4b267a5-b90e-4b3f-80f3-29624198af15","remote_addr":"[fd00:1122:3344:101::b]:59787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:55.175013176Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38664"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:55.175104828Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"42bf443a-2b4c-467d-aeba-cb1cbc1cf0d1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38664","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:25:57.598378129Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"1a04f744-bf8f-47be-923c-245c2ea421cc","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6661,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.057691796Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62614"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.057778468Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c412fc3c-462e-4531-b864-0c92a01e1d5c","remote_addr":"[fd00:1122:3344:101::c]:62614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.074964618Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"57ffab05-12d9-475e-972e-e131faa62548","remote_addr":"[fd00:1122:3344:101::c]:62614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":158,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.116460762Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36826"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.11808393Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.173045759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.250788196Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"12e83295-930c-4f2d-b576-9523bba21ed4","remote_addr":"[fd00:1122:3344:101::c]:36826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":134261,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.251093971Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39783"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.251176152Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64787"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.251372746Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bb1304fb-2d13-4f44-8aaf-74cc107dea1f","remote_addr":"[fd00:1122:3344:101::a]:39783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.251397776Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"20efdfc8-fb39-4b53-b907-5ea0459bdbf7","remote_addr":"[fd00:1122:3344:101::c]:36826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":282,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.252549606Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"3b84a6d2-4512-4093-849c-3f6b0dd7abcd","remote_addr":"[fd00:1122:3344:101::c]:64787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1304,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.262145544Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7b408c16-a1e6-4a17-8192-1d5bd674786c","remote_addr":"[fd00:1122:3344:101::a]:39783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":196,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.297844517Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36984"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.297948608Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.340688914Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.411654342Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"aa3d6275-ef63-4629-a870-8d861534c44c","remote_addr":"[fd00:1122:3344:101::a]:36984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113750,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.411827955Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58583"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.411900877Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52509"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.412854053Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f5da8d18-3bcd-4169-a0e7-995c5614611a","remote_addr":"[fd00:1122:3344:101::b]:52509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":853,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.412879734Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"a9979753-b0e0-4fbb-8c9d-a959584f3619","remote_addr":"[fd00:1122:3344:101::a]:58583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":981,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.412907224Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"035bda39-1fc0-4227-b90b-9a6d040fa64b","remote_addr":"[fd00:1122:3344:101::a]:36984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":941,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.422503842Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1804f8c2-83b7-42d7-a0c3-0b88170eb424","remote_addr":"[fd00:1122:3344:101::b]:52509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":131,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.463124701Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49469"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.463814473Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"066860b6-ce46-4296-92be-17beb06c444a","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":645,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.518238452Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38992"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.518310293Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.556927527Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.611099512Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f28279c7-e3e2-478d-9f8d-2506d2928e7c","remote_addr":"[fd00:1122:3344:101::b]:38992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92811,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:06.613511994Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"58bf75b4-5321-471f-b20e-efb4831bcfa9","remote_addr":"[fd00:1122:3344:101::b]:38992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:25.208879646Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34323"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:25.209518707Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"638685c6-3722-4d3e-841f-4d04ae9a8d92","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34323","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":620,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:27.599237893Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"abde72eb-05de-468b-ab48-d74349d16ec8","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7553,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:35.959446351Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43445"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:35.959530793Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8922128a-b2c8-4e4a-aefb-2c9d5d03cc81","remote_addr":"[fd00:1122:3344:101::c]:43445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:35.977284762Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"29b19bba-dcfc-4064-b902-83acd5d8950b","remote_addr":"[fd00:1122:3344:101::c]:43445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":149,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.23560107Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43269"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.236704069Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"fdd68bc2-1030-44af-8f1d-b5a6e978d1b6","remote_addr":"[fd00:1122:3344:101::c]:43269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1072,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.31072072Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39077"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.310799662Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"84c42b5b-a503-4d7f-a14c-89880152da17","remote_addr":"[fd00:1122:3344:101::a]:39077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.321420557Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35653"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.321536479Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.368924386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.430054132Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e588a4e6-a774-4f30-b281-9fefa6ccdc29","remote_addr":"[fd00:1122:3344:101::c]:35653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108554,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.430312517Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62043"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.430584962Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"af7d1dd4-ecde-47e1-9545-3bf0b83f35a4","remote_addr":"[fd00:1122:3344:101::b]:62043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":185,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.430703194Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"de9ad533-1752-4fb8-8863-c4f664d41680","remote_addr":"[fd00:1122:3344:101::c]:35653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":284,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.431477427Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c5424b56-24a8-4c63-8684-c9e6da45105e","remote_addr":"[fd00:1122:3344:101::a]:39077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":982,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.441213607Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56884"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.441445481Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"fb211a73-b691-4aad-ab16-8f98cf998858","remote_addr":"[fd00:1122:3344:101::a]:56884","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":176,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.441623354Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"66e5cd1f-a195-4dac-9b88-2ae22bfe5c52","remote_addr":"[fd00:1122:3344:101::b]:62043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.443320364Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40323"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.443400085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.483570546Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.547103775Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9885b45f-174b-429e-b5d2-ce874ca0cb82","remote_addr":"[fd00:1122:3344:101::a]:40323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103739,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.547708655Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"10c6d748-8996-43e9-9ed9-8a4081806f7d","remote_addr":"[fd00:1122:3344:101::a]:40323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":227,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.566578185Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64222"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.566786258Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"37c09ffc-6d9d-4d1a-99ae-13e84dc017ae","remote_addr":"[fd00:1122:3344:101::b]:64222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.642658112Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50810"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.642740323Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.683574386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.74055571Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a27f86c8-8699-45cd-b0d7-007f344a0d28","remote_addr":"[fd00:1122:3344:101::b]:50810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97840,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:36.743830507Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9659e628-b5f1-4aa3-9ae8-073f279f7192","remote_addr":"[fd00:1122:3344:101::b]:50810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":673,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:45.612873854Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:55.244978544Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34672"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:55.245054106Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1aa40ab0-f010-4bd8-bc88-ce64f77aeab1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34672","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:26:57.598107542Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"10c11f5a-0c9f-4730-aa46-c507f8480b20","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6197,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:05.975982328Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37407"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:05.976067469Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ba099f23-3a27-4de9-b6f0-e924cd792dbe","remote_addr":"[fd00:1122:3344:101::c]:37407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:05.991938706Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"157e6674-596d-4d62-a2b1-bd387f0c3975","remote_addr":"[fd00:1122:3344:101::c]:37407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1746,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.180239272Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58082"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.180352054Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.224297741Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.28674588Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d30140f1-48d9-42e3-b1d0-7a400dc50489","remote_addr":"[fd00:1122:3344:101::c]:58082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106450,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.287128257Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51506"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.287151567Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52715"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.291789868Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"cafb56b9-2ff7-4774-942f-d3c1458f8fe2","remote_addr":"[fd00:1122:3344:101::a]:51506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":487,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.29188667Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f3907337-e8f8-4a7c-bbab-e08ef29f3bd6","remote_addr":"[fd00:1122:3344:101::c]:52715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":624,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.292012452Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3d8afeb7-d603-4f8b-9e39-a4528ae91310","remote_addr":"[fd00:1122:3344:101::c]:58082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4872,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.296875307Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59301"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.296999149Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"353b590c-da47-4b64-b510-99a5e9c105e5","remote_addr":"[fd00:1122:3344:101::b]:59301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.306409964Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"aa3383d3-eac6-47a1-9194-b5c5e72606d9","remote_addr":"[fd00:1122:3344:101::b]:59301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.33428875Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38663"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.334438453Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d3920b2b-97f9-42d0-acd5-49717da1fbbc","remote_addr":"[fd00:1122:3344:101::a]:38663","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.342959071Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5bd4129d-4696-4c42-bc08-186a66c1fa90","remote_addr":"[fd00:1122:3344:101::a]:51506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":83,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.368908404Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48067"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.369118198Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0d4ecf50-5b5f-4ad7-8ed2-77e771b7ad17","remote_addr":"[fd00:1122:3344:101::b]:48067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.393510143Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41385"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.393605975Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.438904165Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.506997323Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b39c4aee-4f31-4e02-b1e9-54488d349125","remote_addr":"[fd00:1122:3344:101::a]:41385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113424,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.507147256Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47554"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.508715393Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.513076799Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7c88d68e-f283-463e-8af0-74b5afe2d439","remote_addr":"[fd00:1122:3344:101::a]:41385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5702,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.546127366Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.602942047Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"76da3ea9-6a23-4b1d-bea7-67c49ab0b4d2","remote_addr":"[fd00:1122:3344:101::b]:47554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95696,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:06.603373515Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"df85b143-06d7-43f4-be31-ed385b1a7b62","remote_addr":"[fd00:1122:3344:101::b]:47554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:25.279344183Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34573"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:25.279479165Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c9a94d14-5b2d-4120-9755-b495be56b03b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34573","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:27.607509145Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"6d3bea7e-fee7-47a4-8594-823bf39dbe82","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6002,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.14291429Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35371"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.143027952Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7503e821-138c-4f15-933e-22f4e903a530","remote_addr":"[fd00:1122:3344:101::c]:35371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.17616238Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0894b05e-09be-4924-bdc7-44691ae06b4f","remote_addr":"[fd00:1122:3344:101::c]:35371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.2758851Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62820"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.276047913Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"78e070f2-759e-4e39-9b35-55858f202df1","remote_addr":"[fd00:1122:3344:101::c]:62820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.380526546Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50155"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.380584887Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9f0f0dc3-2341-4faa-87b7-8a1dc6caa8bf","remote_addr":"[fd00:1122:3344:101::a]:50155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.390296767Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c110d321-af0b-4f1f-a310-5687ffec3666","remote_addr":"[fd00:1122:3344:101::a]:50155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.40600271Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35343"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.406095542Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.444914229Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.506131248Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d72b806f-8d6a-4b2f-b3f3-4c1070a34c69","remote_addr":"[fd00:1122:3344:101::c]:35343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100069,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.506388782Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61273"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.506416702Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45304"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.506479194Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45526"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.506567125Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.512280025Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"fd6e3619-0ed9-40dc-8589-c3874b0870fd","remote_addr":"[fd00:1122:3344:101::b]:45304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5646,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.512416837Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2f19c8ec-8b38-432b-a0ec-4c402899aaef","remote_addr":"[fd00:1122:3344:101::a]:45526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5772,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.51255383Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d55e8a71-908e-4658-b893-f8717e9896d0","remote_addr":"[fd00:1122:3344:101::c]:35343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5993,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.547677833Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.588313962Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63828"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.588575456Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"21ca42c5-33d5-4f85-9db0-bd081c772d3d","remote_addr":"[fd00:1122:3344:101::b]:63828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":192,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.595800622Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"443e8a02-3876-4dec-9ee5-8c20af4310c0","remote_addr":"[fd00:1122:3344:101::b]:45304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":635,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.604178168Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d2a38ba7-a7fb-43d9-9f86-f8b9786b14f4","remote_addr":"[fd00:1122:3344:101::a]:61273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97712,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.605143795Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"79d09bd0-2145-4367-a009-928ba6773eaa","remote_addr":"[fd00:1122:3344:101::a]:61273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":738,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.692830355Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51797"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.692906416Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.729541316Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.785306719Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8a8f1a10-0d47-406e-8379-b5788055fdef","remote_addr":"[fd00:1122:3344:101::b]:51797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92427,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:36.788193089Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ee484d0d-5d44-45c3-a300-bddcdb7465a5","remote_addr":"[fd00:1122:3344:101::b]:51797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:55.313914515Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54823"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:55.313969896Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8cc7c1fe-5f6f-4f19-901f-3f7c65fbda66","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54823","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:27:57.649293073Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"92bfa993-5c24-4b24-b2a6-98ef4295eae9","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6819,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:05.919364228Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:59604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:05.920210253Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"edc49f59-44f7-4c36-bb13-e127d19a2e2e","remote_addr":"[fd00:1122:3344:101::c]:59604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":825,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:05.927769545Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f68b98fb-2ba7-4330-ae35-90efb65913ff","remote_addr":"[fd00:1122:3344:101::c]:59604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":135,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:05.981028654Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37579"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:05.981177987Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b7b2959f-aa02-4925-9643-0de9f09fe5bb","remote_addr":"[fd00:1122:3344:101::c]:37579","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.028144567Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56763"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.028231378Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.06904833Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.158722805Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2733bff2-aa1b-4daf-9cc8-10c4d8c517de","remote_addr":"[fd00:1122:3344:101::c]:56763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":130506,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.15902006Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59817"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.161614585Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b7e3fd19-253f-4bc2-ac39-f23fa2b62e3b","remote_addr":"[fd00:1122:3344:101::a]:59817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.161708607Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"82478688-8c28-42f3-8093-d33c93db5945","remote_addr":"[fd00:1122:3344:101::c]:56763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":214,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.176867442Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"dd6c1ed0-407e-4f65-9020-d74e5164d2da","remote_addr":"[fd00:1122:3344:101::a]:59817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":210,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.298969582Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55736"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.299156615Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d49cb354-0e63-4c55-be2c-c1786b7dd4a5","remote_addr":"[fd00:1122:3344:101::a]:55736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":145,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.321311402Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52886"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.321416794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.36020853Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.419314902Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b69ce2f4-a324-43b3-bfab-96dbdf7138be","remote_addr":"[fd00:1122:3344:101::a]:52886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97944,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.419547196Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54883"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.419721579Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"baddd91c-5ba9-4d45-b9ce-6ae9fc74a34f","remote_addr":"[fd00:1122:3344:101::b]:54883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.419838031Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"35e9f2db-cb7c-43f9-9caa-c3bf97f56ba3","remote_addr":"[fd00:1122:3344:101::a]:52886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":236,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.42894979Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"213f524b-d32f-41c7-9c66-93dafa1cab6e","remote_addr":"[fd00:1122:3344:101::b]:54883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":132,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.471979161Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40948"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.472177014Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2aa5182b-a8e9-48b1-be05-d0bce000be99","remote_addr":"[fd00:1122:3344:101::b]:40948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.486234849Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44986"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.486342571Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.524036799Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.580570566Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ea28feba-2432-4cf7-bd74-38ce3a2e956e","remote_addr":"[fd00:1122:3344:101::b]:44986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94262,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:06.58480906Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d8916a5c-8397-43c3-9831-8a27daaea353","remote_addr":"[fd00:1122:3344:101::b]:44986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":218,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:25.347665163Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35937"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:25.347742664Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c87a89e8-45d1-4e48-9497-f9f2353ffdd7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35937","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:27.649224181Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"53cc0bee-67a7-4d75-9c27-a612d50d9ec1","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6512,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:35.944789781Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62913"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:35.945753918Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b921586e-c989-49ed-b1f9-7ab493bb0cd0","remote_addr":"[fd00:1122:3344:101::c]:62913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":927,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:35.956700399Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7e64a682-2d1f-4837-98c4-79ac0007ede2","remote_addr":"[fd00:1122:3344:101::c]:62913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":136,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.213461339Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57885"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.213688513Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"68e4e108-ba69-4dff-b090-ee1ead4a0c7f","remote_addr":"[fd00:1122:3344:101::c]:57885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.296151332Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56966"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.296279844Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7845c15a-f5f1-420f-b788-cf1b0cbf7eb2","remote_addr":"[fd00:1122:3344:101::a]:56966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.306747656Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"35ab56a1-9f2c-485c-a101-739ae82c2805","remote_addr":"[fd00:1122:3344:101::a]:56966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.369491621Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42437"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.369595953Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.410280953Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.471300538Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"095d3814-6b25-4ae2-8eef-2d5c2692cfa9","remote_addr":"[fd00:1122:3344:101::c]:42437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101754,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.471631664Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48080"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.471645814Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39713"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.471677144Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44672"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.471820007Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4a28a355-cb55-46ae-a77e-4bb087ea9c6f","remote_addr":"[fd00:1122:3344:101::c]:42437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":254,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.471967149Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"82b580bc-a40d-42ab-99d0-428e7737c2bd","remote_addr":"[fd00:1122:3344:101::b]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":223,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.472024Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.472068081Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8187c446-c029-456b-abd6-935ca96ea24d","remote_addr":"[fd00:1122:3344:101::a]:44672","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":285,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.480785673Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ac8ed03c-dadc-47e3-a1d5-a1adf92f859b","remote_addr":"[fd00:1122:3344:101::b]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":171,"response_code":"204"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.512372634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.538417599Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42271"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.538747245Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b11e6c75-24d6-45f5-9624-06a7a4d99a55","remote_addr":"[fd00:1122:3344:101::b]:42271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":245,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.569878788Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ba396461-25ad-4a9d-b9e7-f1cbab9394fc","remote_addr":"[fd00:1122:3344:101::a]:48080","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98005,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.570278455Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"aedf5511-eeba-4c87-b951-4366b54dfbfc","remote_addr":"[fd00:1122:3344:101::a]:48080","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":156,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.587417694Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.587542796Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.625813984Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.680374166Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2457ed80-8c2a-41dc-bd6e-350613f98d64","remote_addr":"[fd00:1122:3344:101::b]:59200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92874,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:36.684944746Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fbf88ee7-3a5c-48a4-bdb9-12d6ca53a223","remote_addr":"[fd00:1122:3344:101::b]:59200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:55.382594291Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34496"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:55.382737114Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a0500ee4-0aff-4bbb-9ff0-4d29d89ab865","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34496","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:28:57.649006316Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"fce71967-dba4-4808-9c34-916da4c6e03d","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5774,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:05.958215924Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42156"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:05.959094129Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1eb96f46-469c-49fc-a7fe-2bebdcf90f57","remote_addr":"[fd00:1122:3344:101::c]:42156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":846,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:05.968964732Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"fd06c349-a006-4760-b070-28a07c52ac58","remote_addr":"[fd00:1122:3344:101::c]:42156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.074218958Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37291"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.075118704Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"a03a2922-bccf-4a34-b028-2593d1d03f7e","remote_addr":"[fd00:1122:3344:101::c]:37291","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":877,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.123429057Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65484"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.123516629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.171274842Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.241193012Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3341a828-4de3-4887-b038-95a457b8a5e1","remote_addr":"[fd00:1122:3344:101::c]:65484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117710,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.241401405Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60832"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.242410133Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"2e4bb02c-f33d-4cce-8f15-937ba6718aef","remote_addr":"[fd00:1122:3344:101::a]:60832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":950,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.242510635Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"027bd71e-8377-4549-9932-8cebc5f84d14","remote_addr":"[fd00:1122:3344:101::c]:65484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":983,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.280148471Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46943"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.280249723Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c7a4fbc1-4119-41fb-95aa-ed3733398cf4","remote_addr":"[fd00:1122:3344:101::b]:46943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":50,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.289776579Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1e8a3302-54c6-4932-9a6a-e8b94e6102c9","remote_addr":"[fd00:1122:3344:101::b]:46943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":730,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.29093891Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64268"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.291104903Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"479d5608-788b-41d5-a60e-453e128bacfe","remote_addr":"[fd00:1122:3344:101::a]:64268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.301135768Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"50c7c407-9466-4bf7-8f64-22b48128570c","remote_addr":"[fd00:1122:3344:101::a]:60832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.366003389Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54606"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.366118522Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.404718505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.462012265Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1ad6860f-2adb-443d-8fca-dfb59f4ebfa4","remote_addr":"[fd00:1122:3344:101::a]:54606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95949,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.462167158Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50335"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.462264659Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37266"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.462471363Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d91637b1-fafd-4e36-8904-ac7dab5418cb","remote_addr":"[fd00:1122:3344:101::b]:50335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":235,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.462630955Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f4aea006-f951-452b-b8bc-a9e2541290d1","remote_addr":"[fd00:1122:3344:101::a]:54606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":296,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.463217786Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.501480233Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.558700952Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f5e56bd0-2054-49c2-b2e1-8432596f9182","remote_addr":"[fd00:1122:3344:101::b]:37266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96238,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:06.55915461Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8048965b-946d-44ee-aa54-da3500d61c6e","remote_addr":"[fd00:1122:3344:101::b]:37266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:25.414544868Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48410"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:25.414638239Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0def75ff-f8e6-4516-962e-7c15ff2c3a1b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48410","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:27.6498687Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"fe32ff0c-4ddc-498c-95f4-701e60529191","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6467,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.171441493Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.17352234Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"888a5109-91fb-42a8-bea9-15e57e0716bc","remote_addr":"[fd00:1122:3344:101::c]:65014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2035,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.23428969Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5f5e9573-61ee-41e0-a993-de9b412d1613","remote_addr":"[fd00:1122:3344:101::c]:65014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1707,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.306678253Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52132"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.306753834Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b1b69cda-a5f4-42c9-aab4-bdcea19922cd","remote_addr":"[fd00:1122:3344:101::a]:52132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.373932986Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40458"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.374225812Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f4ac1163-ec97-4d67-b586-8d773d1c0764","remote_addr":"[fd00:1122:3344:101::a]:40458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":257,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.383688927Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"19932347-717a-4c7a-951c-56ac843be351","remote_addr":"[fd00:1122:3344:101::a]:52132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.421003138Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34372"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.421195681Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"52e00cbd-a576-4d10-9880-4fee2b5614ff","remote_addr":"[fd00:1122:3344:101::c]:34372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.422878651Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65307"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.423681545Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.466172306Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.52945673Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9e1f2686-68ba-46f1-96e1-85de5a07801e","remote_addr":"[fd00:1122:3344:101::c]:65307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106493,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.529779746Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3cb6668e-9e6a-49f9-a698-71c482ce40d5","remote_addr":"[fd00:1122:3344:101::c]:65307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.544456682Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59748"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.544657665Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3bdb4da7-0c47-4170-b420-ac824803d52b","remote_addr":"[fd00:1122:3344:101::b]:59748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.559874621Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"78f1dbf6-2e6e-4aed-8558-7929bd174eef","remote_addr":"[fd00:1122:3344:101::b]:59748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600408748Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53582"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600623752Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600653313Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600701163Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600713843Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600725274Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600739484Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600768804Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.600786685Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3bbcabdf-84fe-4cd4-809c-29233874f913","remote_addr":"[fd00:1122:3344:101::a]:53582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":290,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.614954282Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62103"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.615896968Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0bea79ff-2041-456c-b3cc-aff3a5416750","remote_addr":"[fd00:1122:3344:101::b]:62103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":870,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636089651Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45450"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636296815Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636322425Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636361005Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636385976Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636409416Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636429287Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636459757Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.636472298Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"bb6fb7fe-dab4-460c-9580-942ad3864dfd","remote_addr":"[fd00:1122:3344:101::c]:45450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":326,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.649892662Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61787"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.650014854Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.688730529Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.746519728Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"89e639bc-9b25-4bab-a7c9-1ee43fd6e968","remote_addr":"[fd00:1122:3344:101::b]:61787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96551,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.751891271Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"923d0aba-4c84-4e50-8854-0225d0d37f9c","remote_addr":"[fd00:1122:3344:101::b]:61787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757164823Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36007"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757420148Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 18a2a554-1ac2-4de0-9094-6a7f88c733cb, kind: Service { id: efb05780-75dc-4711-84b3-13f5e729e36e }, name: Name(\"external-dns-efb05780-75dc-4711-84b3-13f5e729e36e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 222])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5a0a2692-5d18-4240-a52f-42de8fb0d799, kind: Service { id: 22ec8121-6583-4054-9535-d2454e5c4dec }, name: Name(\"external-dns-22ec8121-6583-4054-9535-d2454e5c4dec\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 195fc078-1d03-44a4-9988-9d5e830ac06d, kind: Service { id: 19a73de5-6a55-413b-9592-dba813ea235c }, name: Name(\"nexus-19a73de5-6a55-413b-9592-dba813ea235c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 179])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b1fc1065-f028-43e0-906e-dbfb2b001a34, kind: Service { id: 9970237b-c8a0-4fe2-a597-d8dfdeab350e }, name: Name(\"nexus-9970237b-c8a0-4fe2-a597-d8dfdeab350e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 96])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: d56ab745-3d48-480d-ac5c-97da6e8d2eb9, kind: Service { id: 1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc }, name: Name(\"nexus-1cf5e6e1-7d8d-4b5c-9893-51dfb7a42cdc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 234])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757465159Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757500839Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.75752209Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.75755567Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757594171Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757610601Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.757629041Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ed71a5a3-2d08-4347-99bf-fb09de9f5d69","remote_addr":"[fd00:1122:3344:101::b]:36007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":390,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.827113874Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34383"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.827885307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.863277085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.916145117Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9fb77d76-e896-4168-b973-13a2b4a6ddb2","remote_addr":"[fd00:1122:3344:101::a]:34383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88987,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:36.916469572Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6f2260bb-4d9b-49e7-aaae-9f913bdeada3","remote_addr":"[fd00:1122:3344:101::a]:34383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:55.448392357Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54100"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:55.448466968Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"985a7b78-8f1e-461a-8ab9-9fd183c05401","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54100","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:29:57.64938084Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"703458f4-7a09-45d6-bad2-b9fab1986a21","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6477,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.018745227Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40085"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.018844439Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"78b5e84b-fe12-46be-aa6c-ec899d8fafc9","remote_addr":"[fd00:1122:3344:101::c]:40085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.039097293Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51491"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.039269445Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"c508febd-74da-4ed4-8754-a04d17b37294","remote_addr":"[fd00:1122:3344:101::c]:51491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.047074412Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"035c5cdc-66cd-4be2-aa9b-6d7634becf5b","remote_addr":"[fd00:1122:3344:101::c]:40085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.10430948Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49731"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.104759188Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3f705943-2c91-453c-b353-d76007f932fe","remote_addr":"[fd00:1122:3344:101::a]:49731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":393,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.114193203Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0e824d7e-fb64-4d7b-9015-fcfdd6ec5ca2","remote_addr":"[fd00:1122:3344:101::a]:49731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.133334047Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35786"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.133461259Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.177511428Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.241660697Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f126633b-98fd-4072-942e-9a04501c1d56","remote_addr":"[fd00:1122:3344:101::c]:35786","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108253,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.241954442Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61842"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.242252367Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"393d10a5-628b-488d-a4cb-55effa5e8a6d","remote_addr":"[fd00:1122:3344:101::c]:35786","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":263,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.242322739Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0334ef94-754f-451e-9d56-c140c4af166a","remote_addr":"[fd00:1122:3344:101::a]:61842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":284,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.297923459Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33719"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.29803156Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.341009401Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.398993962Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"18bd566b-3895-42bd-bc80-56e8af190769","remote_addr":"[fd00:1122:3344:101::a]:33719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101027,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.399123515Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63280"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.399210736Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"2f54ae02-d8c5-4908-bd87-99b6e54e2de5","remote_addr":"[fd00:1122:3344:101::b]:63280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.399392949Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"549edbff-6f5f-4099-b074-9a94bb73850b","remote_addr":"[fd00:1122:3344:101::a]:33719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.40915825Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8b6a8872-2785-40da-95fa-a262ef698172","remote_addr":"[fd00:1122:3344:101::b]:63280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":193,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.416440727Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36759"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.416522988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.456715049Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.515101348Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b2d1bdc6-7e46-43c4-9cba-36694b286960","remote_addr":"[fd00:1122:3344:101::b]:36759","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98599,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.515581507Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fddf3449-56f3-40be-a7fc-a86c5a8265fe","remote_addr":"[fd00:1122:3344:101::b]:36759","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.559402151Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52868"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:06.560314507Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8585e62d-96c1-49dc-9406-a1774338f12c","remote_addr":"[fd00:1122:3344:101::b]:52868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":850,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:25.481672806Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51319"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:25.481739257Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"12317747-66cf-42c1-ba62-0ecd251ec2a4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51319","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:27.73260397Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"83dd5ed6-40bd-49ce-a511-7feaabcd45c1","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6354,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.074898226Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44515"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.074999138Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"454a628b-91d9-417f-bc76-54f4509c7356","remote_addr":"[fd00:1122:3344:101::c]:44515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.092599215Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6a2a8005-c818-42ac-8b52-408d7235c140","remote_addr":"[fd00:1122:3344:101::c]:44515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":650,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.29746721Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43878"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.297643873Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"7e5d3018-f03b-4f00-925f-c380499ce94a","remote_addr":"[fd00:1122:3344:101::c]:43878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":150,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.350171339Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34172"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.350251311Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.389472475Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.446723484Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"54a7ce61-f2ba-42b5-aea9-7378ed49d1f7","remote_addr":"[fd00:1122:3344:101::c]:34172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96506,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.446927437Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44243"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.44708881Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e1b75cac-ce78-4533-b16f-b48b45438ae3","remote_addr":"[fd00:1122:3344:101::c]:34172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.447315914Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f1864a53-ba8e-4527-a1e8-be6f3ef19b54","remote_addr":"[fd00:1122:3344:101::a]:44243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.460314641Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"17c62d4f-573b-42ab-add2-9d5bbe852097","remote_addr":"[fd00:1122:3344:101::a]:44243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.460489744Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36057"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.460627497Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1850c45e-2c0d-4b36-b592-b92bf38718d8","remote_addr":"[fd00:1122:3344:101::b]:36057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.469129115Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"cd3ac219-76b4-49b4-9182-a2917829a815","remote_addr":"[fd00:1122:3344:101::b]:36057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":62,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.640299412Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44136"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.640466685Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"5861e151-cf56-4e1b-a434-d9918c6ee01a","remote_addr":"[fd00:1122:3344:101::a]:44136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.64423234Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41148"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.645067215Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.681638083Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.736315247Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7766f541-8770-418e-8d08-382c7c4e808f","remote_addr":"[fd00:1122:3344:101::a]:41148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92035,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.736447309Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41708"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.736675063Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b4965f9d-a7df-4b56-9e73-7a3f0ff1ef08","remote_addr":"[fd00:1122:3344:101::b]:41708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":179,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.736810746Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fa22cb8e-75d6-4c9b-ac6d-9121d582f25c","remote_addr":"[fd00:1122:3344:101::a]:41148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":209,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.796275623Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54500"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.796360545Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.832271101Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.894046958Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c5bae899-38d8-487c-9a18-1c55461afa9e","remote_addr":"[fd00:1122:3344:101::b]:54500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97710,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:36.89698181Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e75a3e18-3c0d-42c3-8370-3211cb12197f","remote_addr":"[fd00:1122:3344:101::b]:54500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:55.515510736Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49960"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:55.515610007Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bc2e8973-e1e9-4aec-a460-0f8a58613497","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49960","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:30:57.732909165Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"618c76a8-9d3e-455f-a9e9-23d4e6fa94ca","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6698,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:05.968201253Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60415"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:05.968334305Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"94fa6372-f843-4090-83ec-c87ff09ae218","remote_addr":"[fd00:1122:3344:101::c]:60415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:05.977034247Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4fbd5307-158f-475f-b3d9-0fdfa8916bc7","remote_addr":"[fd00:1122:3344:101::c]:60415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":135,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.067699769Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46936"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.068781648Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d2ae98ae-4da9-47ca-914a-52ad3c3118f8","remote_addr":"[fd00:1122:3344:101::c]:46936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1027,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.102540357Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35349"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.102659699Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.156795934Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.244537035Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"cf1f1fb6-cdc5-45ba-a8ba-ade207466abc","remote_addr":"[fd00:1122:3344:101::c]:35349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141924,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.244680198Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44456"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.244795709Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"52795b2b-0ca6-4c29-bcff-366aea2a46a1","remote_addr":"[fd00:1122:3344:101::a]:44456","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.244961432Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fa20ce98-b54c-4746-8bfa-5cae0a0e8b70","remote_addr":"[fd00:1122:3344:101::c]:35349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.255014598Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"531b8b55-a892-400b-b9f8-da6f20b0909d","remote_addr":"[fd00:1122:3344:101::a]:44456","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":163,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.299706027Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50736"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.299811159Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7595890a-320b-49ef-ade2-dbb422428ae5","remote_addr":"[fd00:1122:3344:101::b]:50736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.309411027Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"558624ce-d228-4948-80d6-65bda0da8d34","remote_addr":"[fd00:1122:3344:101::b]:50736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.314524486Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55402"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.315191128Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.356537689Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.414132664Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"617d7642-c1cb-4048-b8d8-d460216fce23","remote_addr":"[fd00:1122:3344:101::a]:55402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99554,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.414353198Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59969"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.414416079Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36964"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.41446663Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56815"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.414680104Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"5b940781-9197-475f-b653-4f966b62c5d2","remote_addr":"[fd00:1122:3344:101::b]:59969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":245,"response_code":"200"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.415449277Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.415556419Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8795fac5-519e-46f0-8dc4-fca78e975de9","remote_addr":"[fd00:1122:3344:101::a]:55402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":994,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.415567019Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8c47ab39-f627-43af-b1f9-f1a9e0e88edb","remote_addr":"[fd00:1122:3344:101::a]:56815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":989,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.455774881Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.512666043Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c2eb9b16-b570-40f1-8619-3702a950dc30","remote_addr":"[fd00:1122:3344:101::b]:36964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98047,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:06.513095621Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"db978a2d-315b-4d7d-a435-2979bdc48945","remote_addr":"[fd00:1122:3344:101::b]:36964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.09535364Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45198"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.096072382Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b47000bc-2e23-4ea1-9dea-ce62288bb395","remote_addr":"[fd00:1122:3344:101::a]:45198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":698,"response_code":"200"}
{"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.096310937Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:778","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(13144484) }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.096340757Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"PUT","req_id":"bf78f871-eb8c-4dfa-9642-5a7639f25ff8","remote_addr":"[fd00:1122:3344:101::a]:45198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":79,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.204511434Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42602"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.214881215Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:570","metadata":"InstanceMetadata { silo_id: 644a396e-dfc8-46ce-bf81-7b188fb4c5ae, project_id: 448f551c-01bd-448a-b11d-bb2152f07888 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-09-10T21:31:17.204320141Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(12a57104-3b71-45c4-bbff-e90aa18443a8 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-09-10T21:31:16.775086982Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(13144484))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-72a2274e0db\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3e1b11ba-78f3-4240-8b59-f210ec7aeec2, read_only_parent: Some(Volume { block_size: 512, id: f903ed83-c4e7-47a0-9c6f-aa56cf5b69b9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c9a71175-db79-4c0d-9883-9f56258c6594, key: Some(\"H98IPW5wS3ja059Ve7E/e7aaoTscYJAklyGgtxyg1uU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3e1b11ba-78f3-4240-8b59-f210ec7aeec2, key: Some(\"t/Riu90ObTLS9YiOi4pi4lYiPoDWPsZQoPzk0XuVNec=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], cloud_init_bytes: Some(..) }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.214935786Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:603","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.216191078Z","hostname":"sock","pid":1562,"propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:1008","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(13144484))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 98bf9cb8-479f-49d1-a7b3-ecd0d79e7686, kind: Instance { id: 53ccff73-54b9-4394-92ab-a634606c67a5 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 98, 200])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13144484), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-72a2274e0db\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3e1b11ba-78f3-4240-8b59-f210ec7aeec2, read_only_parent: Some(Volume { block_size: 512, id: f903ed83-c4e7-47a0-9c6f-aa56cf5b69b9, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c9a71175-db79-4c0d-9883-9f56258c6594, key: Some(\"H98IPW5wS3ja059Ve7E/e7aaoTscYJAklyGgtxyg1uU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3e1b11ba-78f3-4240-8b59-f210ec7aeec2, key: Some(\"t/Riu90ObTLS9YiOi4pi4lYiPoDWPsZQoPzk0XuVNec=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::17]:19000\"] } }] } }], cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-09-10T21:31:17.204320141Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.217203046Z","hostname":"sock","pid":1562,"uri":"/vmms/12a57104-3b71-45c4-bbff-e90aa18443a8","method":"PUT","req_id":"6749aeee-1295-4de6-bdbd-75daae18e406","remote_addr":"[fd00:1122:3344:101::a]:42602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":12608,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.299051934Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65319"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.300261685Z","hostname":"sock","pid":1562,"uri":"/vmms/12a57104-3b71-45c4-bbff-e90aa18443a8/state","method":"PUT","req_id":"766bcf51-6f69-4a99-a2f8-a7a5f72386d5","remote_addr":"[fd00:1122:3344:101::a]:65319","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1114,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.300852475Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 254, 98, 200]), slot: 0, vni: Vni { inner: 13144484 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.300881786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.339562441Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:17.364260202Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlInstance0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::1:0'/>\n        <propval type=\"astring\" name=\"listen_port\" value='12400'/>\n        <propval type=\"astring\" name=\"metric_addr\" value='dns'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:18.702673894Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:18.702878138Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 58.844593ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:18.883482809Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 98.738128ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:18.957825226Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 259.909688ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:19.079415578Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 527.677988ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:19.355392713Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 518.524635ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:19.893507762Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 911.390595ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:20.428765161Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 799.436089ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:21.351556572Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 1.229252758s","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:22.161707887Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 658.233546ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:23.401961827Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.092527606Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:1466"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 54.999494ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:24.101707356Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 148.609733ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:24.166524407Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\") failed: Property not found. retry in 291.451353ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:24.323659279Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.624321085Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:1475"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.664358713Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:1499"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.804822115Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:744"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-09-10T21:31:24.807777026Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.807810127Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.807832567Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-09-10T21:31:24.807777026Z }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.807840887Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-09-10T21:31:24.807777026Z }, migration_in: None, migration_out: None }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-09-10T21:31:24.847411307Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.847434278Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.847458948Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-09-10T21:31:24.847411307Z }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:24.847467759Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-09-10T21:31:24.847411307Z }, migration_in: None, migration_out: None }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:25.549947785Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44362"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:25.550089048Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0480f4e6-8dcd-49da-b979-a4cb3f6b2b09","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44362","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:27.732227262Z","hostname":"sock","pid":1562,"uri":"/403254df-bc94-46d0-9b33-45b17d93603a","method":"GET","req_id":"ba94e53f-d93b-4133-9624-c1bc8609fd05","remote_addr":"[fd00:1122:3344:101::d]:60045","local_addr":"[fd00:1122:3344:101::1]:48774","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6254,"response_code":"200"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:31.355349108Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:31.355430629Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:31.388192451Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999547s","next_cleanup":"Instant { tv_sec: 3442, tv_nsec: 905141900 }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:35.966057985Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58559"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:35.967130743Z","hostname":"sock","pid":1562,"uri":"/vmms/12a57104-3b71-45c4-bbff-e90aa18443a8/state","method":"GET","req_id":"4eb67df3-599d-4c19-924e-c3a1800e46d8","remote_addr":"[fd00:1122:3344:101::c]:58559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1034,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.255911682Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64261"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.256007684Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8d146003-2a61-4203-8e55-9f342eb67220","remote_addr":"[fd00:1122:3344:101::c]:64261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.310911032Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49299"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.311107065Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"c94f3d40-950a-4b56-be5a-92e8d8d9ab3b","remote_addr":"[fd00:1122:3344:101::c]:49299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.358960811Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58731"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.359170174Z","hostname":"sock","pid":1562,"uri":"/vmms/12a57104-3b71-45c4-bbff-e90aa18443a8/state","method":"GET","req_id":"04de459f-b5a3-4819-a0b4-b1326ff755cc","remote_addr":"[fd00:1122:3344:101::a]:58731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":170,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.391324355Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39728"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.391515119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.433071513Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.488352468Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9137a0d6-a805-4e35-a7e3-232696b08e31","remote_addr":"[fd00:1122:3344:101::c]:39728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96902,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.488738655Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47170"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.488753635Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48215"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.488829486Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d77081e3-bdb8-4469-9117-0406f6ddc16b","remote_addr":"[fd00:1122:3344:101::c]:39728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":186,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.489168222Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ce47a842-3a3b-48a6-93de-f98a5af76489","remote_addr":"[fd00:1122:3344:101::a]:47170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":274,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.489390486Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2d9cac0f-408a-40c5-8327-f147bfad6738","remote_addr":"[fd00:1122:3344:101::c]:64261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":564,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.490122499Z","hostname":"sock","pid":1562,"uri":"/vmms/12a57104-3b71-45c4-bbff-e90aa18443a8/state","method":"GET","req_id":"2c01a91e-2579-41ac-a667-90ac04c91e88","remote_addr":"[fd00:1122:3344:101::b]:48215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1235,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.53773238Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54044"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.537852652Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"99e17cf8-b8f3-4a1a-a224-ef272f86f567","remote_addr":"[fd00:1122:3344:101::b]:54044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.557347842Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9f50efad-127c-47ad-8ae9-3f183c191fd1","remote_addr":"[fd00:1122:3344:101::b]:54044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.62025555Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46473"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.621231317Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"5f0e0538-e1a7-42cb-a878-853fe30d0aab","remote_addr":"[fd00:1122:3344:101::a]:46473","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":903,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.669509739Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50268"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.669638921Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.708926797Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.762507272Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"013eb50e-b36a-4aa3-aead-f6b4f6d08bc1","remote_addr":"[fd00:1122:3344:101::b]:50268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92916,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.762916279Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64772"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.762943219Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46244"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.76297266Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8bfc7613-b272-4121-a658-cf40beba50f8","remote_addr":"[fd00:1122:3344:101::a]:47170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":253,"response_code":"204"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.763744743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.763813095Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"9176714f-33e9-4700-83cb-a98d4d7f50cb","remote_addr":"[fd00:1122:3344:101::b]:46244","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":771,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.763856185Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"49bae7a2-cd73-424a-8389-ce8394f3aade","remote_addr":"[fd00:1122:3344:101::b]:50268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1000,"response_code":"200"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.802563681Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4038"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.857355177Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8471cd35-4cb0-42f4-b86a-9e1690426548","remote_addr":"[fd00:1122:3344:101::a]:64772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94288,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:36.857760154Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d81f24b6-2c87-48f2-a8e3-a997abfceffc","remote_addr":"[fd00:1122:3344:101::a]:64772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:37.964560075Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41709"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:37.965148625Z","hostname":"sock","pid":1562,"uri":"/vmms/12a57104-3b71-45c4-bbff-e90aa18443a8/state","method":"PUT","req_id":"46231be0-79de-47e6-8311-da01014f8d6e","remote_addr":"[fd00:1122:3344:101::a]:41709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":509,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-09-10T21:31:37.965275677Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:37.965295778Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:37.965329288Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-09-10T21:31:37.965275677Z }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:37.965337248Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-09-10T21:31:37.965275677Z }, migration_in: None, migration_out: None }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-09-10T21:31:38.004315559Z }","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.004333489Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:594"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.004353879Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:616","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-09-10T21:31:38.004315559Z }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.004362599Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:630","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.0043816Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.004564813Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.004613284Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/473e7fed-bfef-4f22-b3d1-ee8652774c28/debug/bundle/zone\", \"/pool/int/d8f3d243-e151-404c-ad0f-38c8ce21760b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/0cb7821c-9c64-4ff9-a1e4-21b3a28ee3f3/crypt/debug\", \"/pool/ext/1d09a0b5-64fe-49da-9f0c-2aeb05367248/crypt/debug\", \"/pool/ext/b41c1bbc-10c8-46e3-9c3a-722fb6b38e83/crypt/debug\", \"/pool/ext/233394d8-3133-44e6-a8c8-baa69c3f73c6/crypt/debug\", \"/pool/ext/a1f44de4-bbe9-4024-862c-8b1b1210d80a/crypt/debug\", \"/pool/ext/eb1402a5-e990-4914-9b22-a55b7b60e97b/crypt/debug\", \"/pool/ext/ea86afc8-d62a-4b61-9643-afc4d22f0535/crypt/debug\", \"/pool/ext/e82c3049-23d1-4237-ad1f-14e37401e51a/crypt/debug\", \"/pool/ext/099b9246-cdfd-4461-86ab-695e2d382672/crypt/debug\"] }","zone_name":"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8"}
{"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:38.005003731Z","hostname":"sock","pid":1562,"component":"kstat-sampler-worker","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":6,"id":"16804845833207710272"}
{"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:38.005287085Z","hostname":"sock","pid":1562,"component":"kstat-sampler-worker","sled_id":"403254df-bc94-46d0-9b33-45b17d93603a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":6,"id":"5193806438767021698"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:38.606843102Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8\", bundle_id: f05f5928-9fb9-460b-a534-04b0148bfa6f }, time_created: 2024-09-10T21:31:38.004856168Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_12a57104-3b71-45c4-bbff-e90aa18443a8","v":0,"name":"SledAgent","level":40,"time":"2024-09-10T21:31:38.606904033Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:843"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.361929984Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.378001995Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(13144484), kind: System }"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.378041365Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(13144484), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.378543694Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-09-10T21:31:40.378517914Z }, migration_in: None, migration_out: None }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.436664288Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:514","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-09-10T21:31:40.378517914Z }, migration_in: None, migration_out: None }","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.442092403Z","hostname":"sock","pid":1562,"instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","propolis_id":"12a57104-3b71-45c4-bbff-e90aa18443a8","instance_id":"53ccff73-54b9-4394-92ab-a634606c67a5","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.966295909Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57906"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.966470932Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"3794c16b-079f-4251-8177-e47da96d8db8","remote_addr":"[fd00:1122:3344:101::c]:57906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":137,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.968019639Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.96809005Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"41635ce4-fd0c-44d7-985a-c23ddb3c551b","remote_addr":"[fd00:1122:3344:101::c]:62958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":21,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:40.975702843Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a662a335-d645-42d3-b84f-7f2a941bd401","remote_addr":"[fd00:1122:3344:101::c]:62958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:41.176269132Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58225"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:41.177027606Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"c0fa8a4a-e721-46c0-addb-a19f59d89719","remote_addr":"[fd00:1122:3344:101::a]:58225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":728,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:42.348768881Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40711"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:42.348873912Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"dd4d2a03-9aab-427b-b596-4dc358295480","remote_addr":"[fd00:1122:3344:101::a]:40711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:42.356464345Z","hostname":"sock","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d037eaf0-884d-4bbe-82f2-7a6df9689ca5","remote_addr":"[fd00:1122:3344:101::a]:40711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"204"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-10T21:31:45.615278695Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}